1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Asynchronous refcounty things |
4 | * |
5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> |
6 | * Copyright 2012 Google, Inc. |
7 | */ |
8 | |
9 | #include <linux/closure.h> |
10 | #include <linux/debugfs.h> |
11 | #include <linux/export.h> |
12 | #include <linux/rcupdate.h> |
13 | #include <linux/seq_file.h> |
14 | #include <linux/sched/debug.h> |
15 | |
16 | static inline void closure_put_after_sub(struct closure *cl, int flags) |
17 | { |
18 | int r = flags & CLOSURE_REMAINING_MASK; |
19 | |
20 | BUG_ON(flags & CLOSURE_GUARD_MASK); |
21 | BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); |
22 | |
23 | if (!r) { |
24 | smp_acquire__after_ctrl_dep(); |
25 | |
26 | cl->closure_get_happened = false; |
27 | |
28 | if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { |
29 | atomic_set(v: &cl->remaining, |
30 | CLOSURE_REMAINING_INITIALIZER); |
31 | closure_queue(cl); |
32 | } else { |
33 | struct closure *parent = cl->parent; |
34 | closure_fn *destructor = cl->fn; |
35 | |
36 | closure_debug_destroy(cl); |
37 | |
38 | if (destructor) |
39 | destructor(&cl->work); |
40 | |
41 | if (parent) |
42 | closure_put(cl: parent); |
43 | } |
44 | } |
45 | } |
46 | |
47 | /* For clearing flags with the same atomic op as a put */ |
48 | void closure_sub(struct closure *cl, int v) |
49 | { |
50 | closure_put_after_sub(cl, flags: atomic_sub_return_release(i: v, v: &cl->remaining)); |
51 | } |
52 | EXPORT_SYMBOL(closure_sub); |
53 | |
54 | /* |
55 | * closure_put - decrement a closure's refcount |
56 | */ |
57 | void closure_put(struct closure *cl) |
58 | { |
59 | closure_put_after_sub(cl, flags: atomic_dec_return_release(v: &cl->remaining)); |
60 | } |
61 | EXPORT_SYMBOL(closure_put); |
62 | |
63 | /* |
64 | * closure_wake_up - wake up all closures on a wait list, without memory barrier |
65 | */ |
66 | void __closure_wake_up(struct closure_waitlist *wait_list) |
67 | { |
68 | struct llist_node *list; |
69 | struct closure *cl, *t; |
70 | struct llist_node *reverse = NULL; |
71 | |
72 | list = llist_del_all(head: &wait_list->list); |
73 | |
74 | /* We first reverse the list to preserve FIFO ordering and fairness */ |
75 | reverse = llist_reverse_order(head: list); |
76 | |
77 | /* Then do the wakeups */ |
78 | llist_for_each_entry_safe(cl, t, reverse, list) { |
79 | closure_set_waiting(cl, f: 0); |
80 | closure_sub(cl, CLOSURE_WAITING + 1); |
81 | } |
82 | } |
83 | EXPORT_SYMBOL(__closure_wake_up); |
84 | |
85 | /** |
86 | * closure_wait - add a closure to a waitlist |
87 | * @waitlist: will own a ref on @cl, which will be released when |
88 | * closure_wake_up() is called on @waitlist. |
89 | * @cl: closure pointer. |
90 | * |
91 | */ |
92 | bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) |
93 | { |
94 | if (atomic_read(v: &cl->remaining) & CLOSURE_WAITING) |
95 | return false; |
96 | |
97 | cl->closure_get_happened = true; |
98 | closure_set_waiting(cl, _RET_IP_); |
99 | atomic_add(i: CLOSURE_WAITING + 1, v: &cl->remaining); |
100 | llist_add(new: &cl->list, head: &waitlist->list); |
101 | |
102 | return true; |
103 | } |
104 | EXPORT_SYMBOL(closure_wait); |
105 | |
106 | struct closure_syncer { |
107 | struct task_struct *task; |
108 | int done; |
109 | }; |
110 | |
111 | static CLOSURE_CALLBACK(closure_sync_fn) |
112 | { |
113 | struct closure *cl = container_of(ws, struct closure, work); |
114 | struct closure_syncer *s = cl->s; |
115 | struct task_struct *p; |
116 | |
117 | rcu_read_lock(); |
118 | p = READ_ONCE(s->task); |
119 | s->done = 1; |
120 | wake_up_process(tsk: p); |
121 | rcu_read_unlock(); |
122 | } |
123 | |
124 | void __sched __closure_sync(struct closure *cl) |
125 | { |
126 | struct closure_syncer s = { .task = current }; |
127 | |
128 | cl->s = &s; |
129 | continue_at(cl, closure_sync_fn, NULL); |
130 | |
131 | while (1) { |
132 | set_current_state(TASK_UNINTERRUPTIBLE); |
133 | if (s.done) |
134 | break; |
135 | schedule(); |
136 | } |
137 | |
138 | __set_current_state(TASK_RUNNING); |
139 | } |
140 | EXPORT_SYMBOL(__closure_sync); |
141 | |
142 | #ifdef CONFIG_DEBUG_CLOSURES |
143 | |
144 | static LIST_HEAD(closure_list); |
145 | static DEFINE_SPINLOCK(closure_list_lock); |
146 | |
147 | void closure_debug_create(struct closure *cl) |
148 | { |
149 | unsigned long flags; |
150 | |
151 | BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); |
152 | cl->magic = CLOSURE_MAGIC_ALIVE; |
153 | |
154 | spin_lock_irqsave(&closure_list_lock, flags); |
155 | list_add(new: &cl->all, head: &closure_list); |
156 | spin_unlock_irqrestore(lock: &closure_list_lock, flags); |
157 | } |
158 | EXPORT_SYMBOL(closure_debug_create); |
159 | |
160 | void closure_debug_destroy(struct closure *cl) |
161 | { |
162 | unsigned long flags; |
163 | |
164 | BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); |
165 | cl->magic = CLOSURE_MAGIC_DEAD; |
166 | |
167 | spin_lock_irqsave(&closure_list_lock, flags); |
168 | list_del(entry: &cl->all); |
169 | spin_unlock_irqrestore(lock: &closure_list_lock, flags); |
170 | } |
171 | EXPORT_SYMBOL(closure_debug_destroy); |
172 | |
173 | static int debug_show(struct seq_file *f, void *data) |
174 | { |
175 | struct closure *cl; |
176 | |
177 | spin_lock_irq(lock: &closure_list_lock); |
178 | |
179 | list_for_each_entry(cl, &closure_list, all) { |
180 | int r = atomic_read(v: &cl->remaining); |
181 | |
182 | seq_printf(m: f, fmt: "%p: %pS -> %pS p %p r %i " , |
183 | cl, (void *) cl->ip, cl->fn, cl->parent, |
184 | r & CLOSURE_REMAINING_MASK); |
185 | |
186 | seq_printf(m: f, fmt: "%s%s\n" , |
187 | test_bit(WORK_STRUCT_PENDING_BIT, |
188 | work_data_bits(&cl->work)) ? "Q" : "" , |
189 | r & CLOSURE_RUNNING ? "R" : "" ); |
190 | |
191 | if (r & CLOSURE_WAITING) |
192 | seq_printf(m: f, fmt: " W %pS\n" , |
193 | (void *) cl->waiting_on); |
194 | |
195 | seq_puts(m: f, s: "\n" ); |
196 | } |
197 | |
198 | spin_unlock_irq(lock: &closure_list_lock); |
199 | return 0; |
200 | } |
201 | |
202 | DEFINE_SHOW_ATTRIBUTE(debug); |
203 | |
204 | static int __init closure_debug_init(void) |
205 | { |
206 | debugfs_create_file(name: "closures" , mode: 0400, NULL, NULL, fops: &debug_fops); |
207 | return 0; |
208 | } |
209 | late_initcall(closure_debug_init) |
210 | |
211 | #endif |
212 | |