1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Asynchronous refcounty things |
4 | * |
5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> |
6 | * Copyright 2012 Google, Inc. |
7 | */ |
8 | |
9 | #include <linux/closure.h> |
10 | #include <linux/debugfs.h> |
11 | #include <linux/export.h> |
12 | #include <linux/rcupdate.h> |
13 | #include <linux/seq_file.h> |
14 | #include <linux/sched/debug.h> |
15 | |
16 | static inline void closure_put_after_sub(struct closure *cl, int flags) |
17 | { |
18 | int r = flags & CLOSURE_REMAINING_MASK; |
19 | |
20 | BUG_ON(flags & CLOSURE_GUARD_MASK); |
21 | BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); |
22 | |
23 | if (!r) { |
24 | smp_acquire__after_ctrl_dep(); |
25 | |
26 | cl->closure_get_happened = false; |
27 | |
28 | if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { |
29 | atomic_set(v: &cl->remaining, |
30 | CLOSURE_REMAINING_INITIALIZER); |
31 | closure_queue(cl); |
32 | } else { |
33 | struct closure *parent = cl->parent; |
34 | closure_fn *destructor = cl->fn; |
35 | |
36 | closure_debug_destroy(cl); |
37 | |
38 | if (destructor) |
39 | destructor(cl); |
40 | |
41 | if (parent) |
42 | closure_put(cl: parent); |
43 | } |
44 | } |
45 | } |
46 | |
47 | /* For clearing flags with the same atomic op as a put */ |
48 | void closure_sub(struct closure *cl, int v) |
49 | { |
50 | closure_put_after_sub(cl, flags: atomic_sub_return_release(i: v, v: &cl->remaining)); |
51 | } |
52 | EXPORT_SYMBOL(closure_sub); |
53 | |
54 | /* |
55 | * closure_put - decrement a closure's refcount |
56 | */ |
57 | void closure_put(struct closure *cl) |
58 | { |
59 | closure_put_after_sub(cl, flags: atomic_dec_return_release(v: &cl->remaining)); |
60 | } |
61 | EXPORT_SYMBOL(closure_put); |
62 | |
63 | /* |
64 | * closure_wake_up - wake up all closures on a wait list, without memory barrier |
65 | */ |
66 | void __closure_wake_up(struct closure_waitlist *wait_list) |
67 | { |
68 | struct llist_node *list; |
69 | struct closure *cl, *t; |
70 | struct llist_node *reverse = NULL; |
71 | |
72 | list = llist_del_all(head: &wait_list->list); |
73 | |
74 | /* We first reverse the list to preserve FIFO ordering and fairness */ |
75 | reverse = llist_reverse_order(head: list); |
76 | |
77 | /* Then do the wakeups */ |
78 | llist_for_each_entry_safe(cl, t, reverse, list) { |
79 | closure_set_waiting(cl, f: 0); |
80 | closure_sub(cl, CLOSURE_WAITING + 1); |
81 | } |
82 | } |
83 | EXPORT_SYMBOL(__closure_wake_up); |
84 | |
85 | /** |
86 | * closure_wait - add a closure to a waitlist |
87 | * @waitlist: will own a ref on @cl, which will be released when |
88 | * closure_wake_up() is called on @waitlist. |
89 | * @cl: closure pointer. |
90 | * |
91 | */ |
92 | bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) |
93 | { |
94 | if (atomic_read(v: &cl->remaining) & CLOSURE_WAITING) |
95 | return false; |
96 | |
97 | cl->closure_get_happened = true; |
98 | closure_set_waiting(cl, _RET_IP_); |
99 | atomic_add(i: CLOSURE_WAITING + 1, v: &cl->remaining); |
100 | llist_add(new: &cl->list, head: &waitlist->list); |
101 | |
102 | return true; |
103 | } |
104 | EXPORT_SYMBOL(closure_wait); |
105 | |
106 | struct closure_syncer { |
107 | struct task_struct *task; |
108 | int done; |
109 | }; |
110 | |
111 | static void closure_sync_fn(struct closure *cl) |
112 | { |
113 | struct closure_syncer *s = cl->s; |
114 | struct task_struct *p; |
115 | |
116 | rcu_read_lock(); |
117 | p = READ_ONCE(s->task); |
118 | s->done = 1; |
119 | wake_up_process(tsk: p); |
120 | rcu_read_unlock(); |
121 | } |
122 | |
123 | void __sched __closure_sync(struct closure *cl) |
124 | { |
125 | struct closure_syncer s = { .task = current }; |
126 | |
127 | cl->s = &s; |
128 | continue_at(cl, closure_sync_fn, NULL); |
129 | |
130 | while (1) { |
131 | set_current_state(TASK_UNINTERRUPTIBLE); |
132 | if (s.done) |
133 | break; |
134 | schedule(); |
135 | } |
136 | |
137 | __set_current_state(TASK_RUNNING); |
138 | } |
139 | EXPORT_SYMBOL(__closure_sync); |
140 | |
141 | #ifdef CONFIG_DEBUG_CLOSURES |
142 | |
143 | static LIST_HEAD(closure_list); |
144 | static DEFINE_SPINLOCK(closure_list_lock); |
145 | |
146 | void closure_debug_create(struct closure *cl) |
147 | { |
148 | unsigned long flags; |
149 | |
150 | BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); |
151 | cl->magic = CLOSURE_MAGIC_ALIVE; |
152 | |
153 | spin_lock_irqsave(&closure_list_lock, flags); |
154 | list_add(new: &cl->all, head: &closure_list); |
155 | spin_unlock_irqrestore(lock: &closure_list_lock, flags); |
156 | } |
157 | EXPORT_SYMBOL(closure_debug_create); |
158 | |
159 | void closure_debug_destroy(struct closure *cl) |
160 | { |
161 | unsigned long flags; |
162 | |
163 | BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); |
164 | cl->magic = CLOSURE_MAGIC_DEAD; |
165 | |
166 | spin_lock_irqsave(&closure_list_lock, flags); |
167 | list_del(entry: &cl->all); |
168 | spin_unlock_irqrestore(lock: &closure_list_lock, flags); |
169 | } |
170 | EXPORT_SYMBOL(closure_debug_destroy); |
171 | |
172 | static int debug_show(struct seq_file *f, void *data) |
173 | { |
174 | struct closure *cl; |
175 | |
176 | spin_lock_irq(lock: &closure_list_lock); |
177 | |
178 | list_for_each_entry(cl, &closure_list, all) { |
179 | int r = atomic_read(v: &cl->remaining); |
180 | |
181 | seq_printf(m: f, fmt: "%p: %pS -> %pS p %p r %i " , |
182 | cl, (void *) cl->ip, cl->fn, cl->parent, |
183 | r & CLOSURE_REMAINING_MASK); |
184 | |
185 | seq_printf(m: f, fmt: "%s%s\n" , |
186 | test_bit(WORK_STRUCT_PENDING_BIT, |
187 | work_data_bits(&cl->work)) ? "Q" : "" , |
188 | r & CLOSURE_RUNNING ? "R" : "" ); |
189 | |
190 | if (r & CLOSURE_WAITING) |
191 | seq_printf(m: f, fmt: " W %pS\n" , |
192 | (void *) cl->waiting_on); |
193 | |
194 | seq_puts(m: f, s: "\n" ); |
195 | } |
196 | |
197 | spin_unlock_irq(lock: &closure_list_lock); |
198 | return 0; |
199 | } |
200 | |
201 | DEFINE_SHOW_ATTRIBUTE(debug); |
202 | |
203 | static int __init closure_debug_init(void) |
204 | { |
205 | debugfs_create_file(name: "closures" , mode: 0400, NULL, NULL, fops: &debug_fops); |
206 | return 0; |
207 | } |
208 | late_initcall(closure_debug_init) |
209 | |
210 | #endif |
211 | |