1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2017 Red Hat. All rights reserved. |
4 | * |
5 | * This file is released under the GPL. |
6 | */ |
7 | |
8 | #include "dm-cache-background-tracker.h" |
9 | |
10 | /*----------------------------------------------------------------*/ |
11 | |
12 | #define DM_MSG_PREFIX "dm-background-tracker" |
13 | |
14 | struct bt_work { |
15 | struct list_head list; |
16 | struct rb_node node; |
17 | struct policy_work work; |
18 | }; |
19 | |
20 | struct background_tracker { |
21 | unsigned int max_work; |
22 | atomic_t pending_promotes; |
23 | atomic_t pending_writebacks; |
24 | atomic_t pending_demotes; |
25 | |
26 | struct list_head issued; |
27 | struct list_head queued; |
28 | struct rb_root pending; |
29 | |
30 | struct kmem_cache *work_cache; |
31 | }; |
32 | |
33 | struct background_tracker *btracker_create(unsigned int max_work) |
34 | { |
35 | struct background_tracker *b = kmalloc(size: sizeof(*b), GFP_KERNEL); |
36 | |
37 | if (!b) { |
38 | DMERR("couldn't create background_tracker" ); |
39 | return NULL; |
40 | } |
41 | |
42 | b->max_work = max_work; |
43 | atomic_set(v: &b->pending_promotes, i: 0); |
44 | atomic_set(v: &b->pending_writebacks, i: 0); |
45 | atomic_set(v: &b->pending_demotes, i: 0); |
46 | |
47 | INIT_LIST_HEAD(list: &b->issued); |
48 | INIT_LIST_HEAD(list: &b->queued); |
49 | |
50 | b->pending = RB_ROOT; |
51 | b->work_cache = KMEM_CACHE(bt_work, 0); |
52 | if (!b->work_cache) { |
53 | DMERR("couldn't create mempool for background work items" ); |
54 | kfree(objp: b); |
55 | b = NULL; |
56 | } |
57 | |
58 | return b; |
59 | } |
60 | EXPORT_SYMBOL_GPL(btracker_create); |
61 | |
62 | void btracker_destroy(struct background_tracker *b) |
63 | { |
64 | struct bt_work *w, *tmp; |
65 | |
66 | BUG_ON(!list_empty(&b->issued)); |
67 | list_for_each_entry_safe (w, tmp, &b->queued, list) { |
68 | list_del(entry: &w->list); |
69 | kmem_cache_free(s: b->work_cache, objp: w); |
70 | } |
71 | |
72 | kmem_cache_destroy(s: b->work_cache); |
73 | kfree(objp: b); |
74 | } |
75 | EXPORT_SYMBOL_GPL(btracker_destroy); |
76 | |
77 | static int cmp_oblock(dm_oblock_t lhs, dm_oblock_t rhs) |
78 | { |
79 | if (from_oblock(b: lhs) < from_oblock(b: rhs)) |
80 | return -1; |
81 | |
82 | if (from_oblock(b: rhs) < from_oblock(b: lhs)) |
83 | return 1; |
84 | |
85 | return 0; |
86 | } |
87 | |
88 | static bool __insert_pending(struct background_tracker *b, |
89 | struct bt_work *nw) |
90 | { |
91 | int cmp; |
92 | struct bt_work *w; |
93 | struct rb_node **new = &b->pending.rb_node, *parent = NULL; |
94 | |
95 | while (*new) { |
96 | w = container_of(*new, struct bt_work, node); |
97 | |
98 | parent = *new; |
99 | cmp = cmp_oblock(lhs: w->work.oblock, rhs: nw->work.oblock); |
100 | if (cmp < 0) |
101 | new = &((*new)->rb_left); |
102 | |
103 | else if (cmp > 0) |
104 | new = &((*new)->rb_right); |
105 | |
106 | else |
107 | /* already present */ |
108 | return false; |
109 | } |
110 | |
111 | rb_link_node(node: &nw->node, parent, rb_link: new); |
112 | rb_insert_color(&nw->node, &b->pending); |
113 | |
114 | return true; |
115 | } |
116 | |
117 | static struct bt_work *__find_pending(struct background_tracker *b, |
118 | dm_oblock_t oblock) |
119 | { |
120 | int cmp; |
121 | struct bt_work *w; |
122 | struct rb_node **new = &b->pending.rb_node; |
123 | |
124 | while (*new) { |
125 | w = container_of(*new, struct bt_work, node); |
126 | |
127 | cmp = cmp_oblock(lhs: w->work.oblock, rhs: oblock); |
128 | if (cmp < 0) |
129 | new = &((*new)->rb_left); |
130 | |
131 | else if (cmp > 0) |
132 | new = &((*new)->rb_right); |
133 | |
134 | else |
135 | break; |
136 | } |
137 | |
138 | return *new ? w : NULL; |
139 | } |
140 | |
141 | |
142 | static void update_stats(struct background_tracker *b, struct policy_work *w, int delta) |
143 | { |
144 | switch (w->op) { |
145 | case POLICY_PROMOTE: |
146 | atomic_add(i: delta, v: &b->pending_promotes); |
147 | break; |
148 | |
149 | case POLICY_DEMOTE: |
150 | atomic_add(i: delta, v: &b->pending_demotes); |
151 | break; |
152 | |
153 | case POLICY_WRITEBACK: |
154 | atomic_add(i: delta, v: &b->pending_writebacks); |
155 | break; |
156 | } |
157 | } |
158 | |
159 | unsigned int btracker_nr_writebacks_queued(struct background_tracker *b) |
160 | { |
161 | return atomic_read(v: &b->pending_writebacks); |
162 | } |
163 | EXPORT_SYMBOL_GPL(btracker_nr_writebacks_queued); |
164 | |
165 | unsigned int btracker_nr_demotions_queued(struct background_tracker *b) |
166 | { |
167 | return atomic_read(v: &b->pending_demotes); |
168 | } |
169 | EXPORT_SYMBOL_GPL(btracker_nr_demotions_queued); |
170 | |
171 | static bool max_work_reached(struct background_tracker *b) |
172 | { |
173 | return atomic_read(v: &b->pending_promotes) + |
174 | atomic_read(v: &b->pending_writebacks) + |
175 | atomic_read(v: &b->pending_demotes) >= b->max_work; |
176 | } |
177 | |
178 | static struct bt_work *alloc_work(struct background_tracker *b) |
179 | { |
180 | if (max_work_reached(b)) |
181 | return NULL; |
182 | |
183 | return kmem_cache_alloc(cachep: b->work_cache, GFP_NOWAIT); |
184 | } |
185 | |
186 | int btracker_queue(struct background_tracker *b, |
187 | struct policy_work *work, |
188 | struct policy_work **pwork) |
189 | { |
190 | struct bt_work *w; |
191 | |
192 | if (pwork) |
193 | *pwork = NULL; |
194 | |
195 | w = alloc_work(b); |
196 | if (!w) |
197 | return -ENOMEM; |
198 | |
199 | memcpy(&w->work, work, sizeof(*work)); |
200 | |
201 | if (!__insert_pending(b, nw: w)) { |
202 | /* |
203 | * There was a race, we'll just ignore this second |
204 | * bit of work for the same oblock. |
205 | */ |
206 | kmem_cache_free(s: b->work_cache, objp: w); |
207 | return -EINVAL; |
208 | } |
209 | |
210 | if (pwork) { |
211 | *pwork = &w->work; |
212 | list_add(new: &w->list, head: &b->issued); |
213 | } else |
214 | list_add(new: &w->list, head: &b->queued); |
215 | update_stats(b, w: &w->work, delta: 1); |
216 | |
217 | return 0; |
218 | } |
219 | EXPORT_SYMBOL_GPL(btracker_queue); |
220 | |
221 | /* |
222 | * Returns -ENODATA if there's no work. |
223 | */ |
224 | int btracker_issue(struct background_tracker *b, struct policy_work **work) |
225 | { |
226 | struct bt_work *w; |
227 | |
228 | if (list_empty(head: &b->queued)) |
229 | return -ENODATA; |
230 | |
231 | w = list_first_entry(&b->queued, struct bt_work, list); |
232 | list_move(list: &w->list, head: &b->issued); |
233 | *work = &w->work; |
234 | |
235 | return 0; |
236 | } |
237 | EXPORT_SYMBOL_GPL(btracker_issue); |
238 | |
239 | void btracker_complete(struct background_tracker *b, |
240 | struct policy_work *op) |
241 | { |
242 | struct bt_work *w = container_of(op, struct bt_work, work); |
243 | |
244 | update_stats(b, w: &w->work, delta: -1); |
245 | rb_erase(&w->node, &b->pending); |
246 | list_del(entry: &w->list); |
247 | kmem_cache_free(s: b->work_cache, objp: w); |
248 | } |
249 | EXPORT_SYMBOL_GPL(btracker_complete); |
250 | |
251 | bool btracker_promotion_already_present(struct background_tracker *b, |
252 | dm_oblock_t oblock) |
253 | { |
254 | return __find_pending(b, oblock) != NULL; |
255 | } |
256 | EXPORT_SYMBOL_GPL(btracker_promotion_already_present); |
257 | |
258 | /*----------------------------------------------------------------*/ |
259 | |