1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * |
4 | * Copyright (C) 2005 Oracle. All rights reserved. |
5 | */ |
6 | |
7 | /* This quorum hack is only here until we transition to some more rational |
8 | * approach that is driven from userspace. Honest. No foolin'. |
9 | * |
10 | * Imagine two nodes lose network connectivity to each other but they're still |
11 | * up and operating in every other way. Presumably a network timeout indicates |
12 | * that a node is broken and should be recovered. They can't both recover each |
13 | * other and both carry on without serialising their access to the file system. |
14 | * They need to decide who is authoritative. Now extend that problem to |
15 | * arbitrary groups of nodes losing connectivity between each other. |
16 | * |
17 | * So we declare that a node which has given up on connecting to a majority |
18 | * of nodes who are still heartbeating will fence itself. |
19 | * |
20 | * There are huge opportunities for races here. After we give up on a node's |
21 | * connection we need to wait long enough to give heartbeat an opportunity |
22 | * to declare the node as truly dead. We also need to be careful with the |
23 | * race between when we see a node start heartbeating and when we connect |
24 | * to it. |
25 | * |
26 | * So nodes that are in this transtion put a hold on the quorum decision |
27 | * with a counter. As they fall out of this transition they drop the count |
28 | * and if they're the last, they fire off the decision. |
29 | */ |
30 | #include <linux/kernel.h> |
31 | #include <linux/workqueue.h> |
32 | #include <linux/reboot.h> |
33 | |
34 | #include "heartbeat.h" |
35 | #include "nodemanager.h" |
36 | #define MLOG_MASK_PREFIX ML_QUORUM |
37 | #include "masklog.h" |
38 | #include "quorum.h" |
39 | |
40 | static struct o2quo_state { |
41 | spinlock_t qs_lock; |
42 | struct work_struct qs_work; |
43 | int qs_pending; |
44 | int qs_heartbeating; |
45 | unsigned long qs_hb_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; |
46 | int qs_connected; |
47 | unsigned long qs_conn_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; |
48 | int qs_holds; |
49 | unsigned long qs_hold_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; |
50 | } o2quo_state; |
51 | |
52 | /* this is horribly heavy-handed. It should instead flip the file |
53 | * system RO and call some userspace script. */ |
54 | static void o2quo_fence_self(void) |
55 | { |
56 | /* panic spins with interrupts enabled. with preempt |
57 | * threads can still schedule, etc, etc */ |
58 | o2hb_stop_all_regions(); |
59 | |
60 | switch (o2nm_single_cluster->cl_fence_method) { |
61 | case O2NM_FENCE_PANIC: |
62 | panic(fmt: "*** ocfs2 is very sorry to be fencing this system by " |
63 | "panicing ***\n" ); |
64 | break; |
65 | default: |
66 | WARN_ON(o2nm_single_cluster->cl_fence_method >= |
67 | O2NM_FENCE_METHODS); |
68 | fallthrough; |
69 | case O2NM_FENCE_RESET: |
70 | printk(KERN_ERR "*** ocfs2 is very sorry to be fencing this " |
71 | "system by restarting ***\n" ); |
72 | emergency_restart(); |
73 | break; |
74 | } |
75 | } |
76 | |
77 | /* Indicate that a timeout occurred on a heartbeat region write. The |
78 | * other nodes in the cluster may consider us dead at that time so we |
79 | * want to "fence" ourselves so that we don't scribble on the disk |
80 | * after they think they've recovered us. This can't solve all |
81 | * problems related to writeout after recovery but this hack can at |
82 | * least close some of those gaps. When we have real fencing, this can |
83 | * go away as our node would be fenced externally before other nodes |
84 | * begin recovery. */ |
85 | void o2quo_disk_timeout(void) |
86 | { |
87 | o2quo_fence_self(); |
88 | } |
89 | |
90 | static void o2quo_make_decision(struct work_struct *work) |
91 | { |
92 | int quorum; |
93 | int lowest_hb, lowest_reachable = 0, fence = 0; |
94 | struct o2quo_state *qs = &o2quo_state; |
95 | |
96 | spin_lock_bh(lock: &qs->qs_lock); |
97 | |
98 | lowest_hb = find_first_bit(addr: qs->qs_hb_bm, O2NM_MAX_NODES); |
99 | if (lowest_hb != O2NM_MAX_NODES) |
100 | lowest_reachable = test_bit(lowest_hb, qs->qs_conn_bm); |
101 | |
102 | mlog(0, "heartbeating: %d, connected: %d, " |
103 | "lowest: %d (%sreachable)\n" , qs->qs_heartbeating, |
104 | qs->qs_connected, lowest_hb, lowest_reachable ? "" : "un" ); |
105 | |
106 | if (!test_bit(o2nm_this_node(), qs->qs_hb_bm) || |
107 | qs->qs_heartbeating == 1) |
108 | goto out; |
109 | |
110 | if (qs->qs_heartbeating & 1) { |
111 | /* the odd numbered cluster case is straight forward -- |
112 | * if we can't talk to the majority we're hosed */ |
113 | quorum = (qs->qs_heartbeating + 1)/2; |
114 | if (qs->qs_connected < quorum) { |
115 | mlog(ML_ERROR, "fencing this node because it is " |
116 | "only connected to %u nodes and %u is needed " |
117 | "to make a quorum out of %u heartbeating nodes\n" , |
118 | qs->qs_connected, quorum, |
119 | qs->qs_heartbeating); |
120 | fence = 1; |
121 | } |
122 | } else { |
123 | /* the even numbered cluster adds the possibility of each half |
124 | * of the cluster being able to talk amongst themselves.. in |
125 | * that case we're hosed if we can't talk to the group that has |
126 | * the lowest numbered node */ |
127 | quorum = qs->qs_heartbeating / 2; |
128 | if (qs->qs_connected < quorum) { |
129 | mlog(ML_ERROR, "fencing this node because it is " |
130 | "only connected to %u nodes and %u is needed " |
131 | "to make a quorum out of %u heartbeating nodes\n" , |
132 | qs->qs_connected, quorum, |
133 | qs->qs_heartbeating); |
134 | fence = 1; |
135 | } |
136 | else if ((qs->qs_connected == quorum) && |
137 | !lowest_reachable) { |
138 | mlog(ML_ERROR, "fencing this node because it is " |
139 | "connected to a half-quorum of %u out of %u " |
140 | "nodes which doesn't include the lowest active " |
141 | "node %u\n" , quorum, qs->qs_heartbeating, |
142 | lowest_hb); |
143 | fence = 1; |
144 | } |
145 | } |
146 | |
147 | out: |
148 | if (fence) { |
149 | spin_unlock_bh(lock: &qs->qs_lock); |
150 | o2quo_fence_self(); |
151 | } else { |
152 | mlog(ML_NOTICE, "not fencing this node, heartbeating: %d, " |
153 | "connected: %d, lowest: %d (%sreachable)\n" , |
154 | qs->qs_heartbeating, qs->qs_connected, lowest_hb, |
155 | lowest_reachable ? "" : "un" ); |
156 | spin_unlock_bh(lock: &qs->qs_lock); |
157 | |
158 | } |
159 | |
160 | } |
161 | |
162 | static void o2quo_set_hold(struct o2quo_state *qs, u8 node) |
163 | { |
164 | assert_spin_locked(&qs->qs_lock); |
165 | |
166 | if (!test_and_set_bit(nr: node, addr: qs->qs_hold_bm)) { |
167 | qs->qs_holds++; |
168 | mlog_bug_on_msg(qs->qs_holds == O2NM_MAX_NODES, |
169 | "node %u\n" , node); |
170 | mlog(0, "node %u, %d total\n" , node, qs->qs_holds); |
171 | } |
172 | } |
173 | |
174 | static void o2quo_clear_hold(struct o2quo_state *qs, u8 node) |
175 | { |
176 | assert_spin_locked(&qs->qs_lock); |
177 | |
178 | if (test_and_clear_bit(nr: node, addr: qs->qs_hold_bm)) { |
179 | mlog(0, "node %u, %d total\n" , node, qs->qs_holds - 1); |
180 | if (--qs->qs_holds == 0) { |
181 | if (qs->qs_pending) { |
182 | qs->qs_pending = 0; |
183 | schedule_work(work: &qs->qs_work); |
184 | } |
185 | } |
186 | mlog_bug_on_msg(qs->qs_holds < 0, "node %u, holds %d\n" , |
187 | node, qs->qs_holds); |
188 | } |
189 | } |
190 | |
191 | /* as a node comes up we delay the quorum decision until we know the fate of |
192 | * the connection. the hold will be droped in conn_up or hb_down. it might be |
193 | * perpetuated by con_err until hb_down. if we already have a conn, we might |
194 | * be dropping a hold that conn_up got. */ |
195 | void o2quo_hb_up(u8 node) |
196 | { |
197 | struct o2quo_state *qs = &o2quo_state; |
198 | |
199 | spin_lock_bh(lock: &qs->qs_lock); |
200 | |
201 | qs->qs_heartbeating++; |
202 | mlog_bug_on_msg(qs->qs_heartbeating == O2NM_MAX_NODES, |
203 | "node %u\n" , node); |
204 | mlog_bug_on_msg(test_bit(node, qs->qs_hb_bm), "node %u\n" , node); |
205 | set_bit(nr: node, addr: qs->qs_hb_bm); |
206 | |
207 | mlog(0, "node %u, %d total\n" , node, qs->qs_heartbeating); |
208 | |
209 | if (!test_bit(node, qs->qs_conn_bm)) |
210 | o2quo_set_hold(qs, node); |
211 | else |
212 | o2quo_clear_hold(qs, node); |
213 | |
214 | spin_unlock_bh(lock: &qs->qs_lock); |
215 | } |
216 | |
217 | /* hb going down releases any holds we might have had due to this node from |
218 | * conn_up, conn_err, or hb_up */ |
219 | void o2quo_hb_down(u8 node) |
220 | { |
221 | struct o2quo_state *qs = &o2quo_state; |
222 | |
223 | spin_lock_bh(lock: &qs->qs_lock); |
224 | |
225 | qs->qs_heartbeating--; |
226 | mlog_bug_on_msg(qs->qs_heartbeating < 0, |
227 | "node %u, %d heartbeating\n" , |
228 | node, qs->qs_heartbeating); |
229 | mlog_bug_on_msg(!test_bit(node, qs->qs_hb_bm), "node %u\n" , node); |
230 | clear_bit(nr: node, addr: qs->qs_hb_bm); |
231 | |
232 | mlog(0, "node %u, %d total\n" , node, qs->qs_heartbeating); |
233 | |
234 | o2quo_clear_hold(qs, node); |
235 | |
236 | spin_unlock_bh(lock: &qs->qs_lock); |
237 | } |
238 | |
239 | /* this tells us that we've decided that the node is still heartbeating |
240 | * even though we've lost it's conn. it must only be called after conn_err |
241 | * and indicates that we must now make a quorum decision in the future, |
242 | * though we might be doing so after waiting for holds to drain. Here |
243 | * we'll be dropping the hold from conn_err. */ |
244 | void o2quo_hb_still_up(u8 node) |
245 | { |
246 | struct o2quo_state *qs = &o2quo_state; |
247 | |
248 | spin_lock_bh(lock: &qs->qs_lock); |
249 | |
250 | mlog(0, "node %u\n" , node); |
251 | |
252 | qs->qs_pending = 1; |
253 | o2quo_clear_hold(qs, node); |
254 | |
255 | spin_unlock_bh(lock: &qs->qs_lock); |
256 | } |
257 | |
258 | /* This is analogous to hb_up. as a node's connection comes up we delay the |
259 | * quorum decision until we see it heartbeating. the hold will be droped in |
260 | * hb_up or hb_down. it might be perpetuated by con_err until hb_down. if |
261 | * it's already heartbeating we might be dropping a hold that conn_up got. |
262 | * */ |
263 | void o2quo_conn_up(u8 node) |
264 | { |
265 | struct o2quo_state *qs = &o2quo_state; |
266 | |
267 | spin_lock_bh(lock: &qs->qs_lock); |
268 | |
269 | qs->qs_connected++; |
270 | mlog_bug_on_msg(qs->qs_connected == O2NM_MAX_NODES, |
271 | "node %u\n" , node); |
272 | mlog_bug_on_msg(test_bit(node, qs->qs_conn_bm), "node %u\n" , node); |
273 | set_bit(nr: node, addr: qs->qs_conn_bm); |
274 | |
275 | mlog(0, "node %u, %d total\n" , node, qs->qs_connected); |
276 | |
277 | if (!test_bit(node, qs->qs_hb_bm)) |
278 | o2quo_set_hold(qs, node); |
279 | else |
280 | o2quo_clear_hold(qs, node); |
281 | |
282 | spin_unlock_bh(lock: &qs->qs_lock); |
283 | } |
284 | |
285 | /* we've decided that we won't ever be connecting to the node again. if it's |
286 | * still heartbeating we grab a hold that will delay decisions until either the |
287 | * node stops heartbeating from hb_down or the caller decides that the node is |
288 | * still up and calls still_up */ |
289 | void o2quo_conn_err(u8 node) |
290 | { |
291 | struct o2quo_state *qs = &o2quo_state; |
292 | |
293 | spin_lock_bh(lock: &qs->qs_lock); |
294 | |
295 | if (test_bit(node, qs->qs_conn_bm)) { |
296 | qs->qs_connected--; |
297 | mlog_bug_on_msg(qs->qs_connected < 0, |
298 | "node %u, connected %d\n" , |
299 | node, qs->qs_connected); |
300 | |
301 | clear_bit(nr: node, addr: qs->qs_conn_bm); |
302 | |
303 | if (test_bit(node, qs->qs_hb_bm)) |
304 | o2quo_set_hold(qs, node); |
305 | } |
306 | |
307 | mlog(0, "node %u, %d total\n" , node, qs->qs_connected); |
308 | |
309 | |
310 | spin_unlock_bh(lock: &qs->qs_lock); |
311 | } |
312 | |
313 | void o2quo_init(void) |
314 | { |
315 | struct o2quo_state *qs = &o2quo_state; |
316 | |
317 | spin_lock_init(&qs->qs_lock); |
318 | INIT_WORK(&qs->qs_work, o2quo_make_decision); |
319 | } |
320 | |
321 | void o2quo_exit(void) |
322 | { |
323 | struct o2quo_state *qs = &o2quo_state; |
324 | |
325 | flush_work(work: &qs->qs_work); |
326 | } |
327 | |