1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_RCULIST_H
3#define _LINUX_RCULIST_H
4
5#ifdef __KERNEL__
6
7/*
8 * RCU-protected list version
9 */
10#include <linux/list.h>
11#include <linux/rcupdate.h>
12
13/*
14 * Why is there no list_empty_rcu()? Because list_empty() serves this
15 * purpose. The list_empty() function fetches the RCU-protected pointer
16 * and compares it to the address of the list head, but neither dereferences
17 * this pointer itself nor provides this pointer to the caller. Therefore,
18 * it is not necessary to use rcu_dereference(), so that list_empty() can
19 * be used anywhere you would want to use a list_empty_rcu().
20 */
21
22/*
23 * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
24 * @list: list to be initialized
25 *
26 * You should instead use INIT_LIST_HEAD() for normal initialization and
27 * cleanup tasks, when readers have no access to the list being initialized.
28 * However, if the list being initialized is visible to readers, you
29 * need to keep the compiler from being too mischievous.
30 */
31static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
32{
33 WRITE_ONCE(list->next, list);
34 WRITE_ONCE(list->prev, list);
35}
36
37/*
38 * return the ->next pointer of a list_head in an rcu safe
39 * way, we must not access it directly
40 */
41#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
42
43/*
44 * Insert a new entry between two known consecutive entries.
45 *
46 * This is only for internal list manipulation where we know
47 * the prev/next entries already!
48 */
49static inline void __list_add_rcu(struct list_head *new,
50 struct list_head *prev, struct list_head *next)
51{
52 if (!__list_add_valid(new, prev, next))
53 return;
54
55 new->next = next;
56 new->prev = prev;
57 rcu_assign_pointer(list_next_rcu(prev), new);
58 next->prev = new;
59}
60
61/**
62 * list_add_rcu - add a new entry to rcu-protected list
63 * @new: new entry to be added
64 * @head: list head to add it after
65 *
66 * Insert a new entry after the specified head.
67 * This is good for implementing stacks.
68 *
69 * The caller must take whatever precautions are necessary
70 * (such as holding appropriate locks) to avoid racing
71 * with another list-mutation primitive, such as list_add_rcu()
72 * or list_del_rcu(), running on this same list.
73 * However, it is perfectly legal to run concurrently with
74 * the _rcu list-traversal primitives, such as
75 * list_for_each_entry_rcu().
76 */
77static inline void list_add_rcu(struct list_head *new, struct list_head *head)
78{
79 __list_add_rcu(new, head, head->next);
80}
81
82/**
83 * list_add_tail_rcu - add a new entry to rcu-protected list
84 * @new: new entry to be added
85 * @head: list head to add it before
86 *
87 * Insert a new entry before the specified head.
88 * This is useful for implementing queues.
89 *
90 * The caller must take whatever precautions are necessary
91 * (such as holding appropriate locks) to avoid racing
92 * with another list-mutation primitive, such as list_add_tail_rcu()
93 * or list_del_rcu(), running on this same list.
94 * However, it is perfectly legal to run concurrently with
95 * the _rcu list-traversal primitives, such as
96 * list_for_each_entry_rcu().
97 */
98static inline void list_add_tail_rcu(struct list_head *new,
99 struct list_head *head)
100{
101 __list_add_rcu(new, head->prev, head);
102}
103
104/**
105 * list_del_rcu - deletes entry from list without re-initialization
106 * @entry: the element to delete from the list.
107 *
108 * Note: list_empty() on entry does not return true after this,
109 * the entry is in an undefined state. It is useful for RCU based
110 * lockfree traversal.
111 *
112 * In particular, it means that we can not poison the forward
113 * pointers that may still be used for walking the list.
114 *
115 * The caller must take whatever precautions are necessary
116 * (such as holding appropriate locks) to avoid racing
117 * with another list-mutation primitive, such as list_del_rcu()
118 * or list_add_rcu(), running on this same list.
119 * However, it is perfectly legal to run concurrently with
120 * the _rcu list-traversal primitives, such as
121 * list_for_each_entry_rcu().
122 *
123 * Note that the caller is not permitted to immediately free
124 * the newly deleted entry. Instead, either synchronize_rcu()
125 * or call_rcu() must be used to defer freeing until an RCU
126 * grace period has elapsed.
127 */
128static inline void list_del_rcu(struct list_head *entry)
129{
130 __list_del_entry(entry);
131 entry->prev = LIST_POISON2;
132}
133
134/**
135 * hlist_del_init_rcu - deletes entry from hash list with re-initialization
136 * @n: the element to delete from the hash list.
137 *
138 * Note: list_unhashed() on the node return true after this. It is
139 * useful for RCU based read lockfree traversal if the writer side
140 * must know if the list entry is still hashed or already unhashed.
141 *
142 * In particular, it means that we can not poison the forward pointers
143 * that may still be used for walking the hash list and we can only
144 * zero the pprev pointer so list_unhashed() will return true after
145 * this.
146 *
147 * The caller must take whatever precautions are necessary (such as
148 * holding appropriate locks) to avoid racing with another
149 * list-mutation primitive, such as hlist_add_head_rcu() or
150 * hlist_del_rcu(), running on this same list. However, it is
151 * perfectly legal to run concurrently with the _rcu list-traversal
152 * primitives, such as hlist_for_each_entry_rcu().
153 */
154static inline void hlist_del_init_rcu(struct hlist_node *n)
155{
156 if (!hlist_unhashed(n)) {
157 __hlist_del(n);
158 n->pprev = NULL;
159 }
160}
161
162/**
163 * list_replace_rcu - replace old entry by new one
164 * @old : the element to be replaced
165 * @new : the new element to insert
166 *
167 * The @old entry will be replaced with the @new entry atomically.
168 * Note: @old should not be empty.
169 */
170static inline void list_replace_rcu(struct list_head *old,
171 struct list_head *new)
172{
173 new->next = old->next;
174 new->prev = old->prev;
175 rcu_assign_pointer(list_next_rcu(new->prev), new);
176 new->next->prev = new;
177 old->prev = LIST_POISON2;
178}
179
180/**
181 * __list_splice_init_rcu - join an RCU-protected list into an existing list.
182 * @list: the RCU-protected list to splice
183 * @prev: points to the last element of the existing list
184 * @next: points to the first element of the existing list
185 * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
186 *
187 * The list pointed to by @prev and @next can be RCU-read traversed
188 * concurrently with this function.
189 *
190 * Note that this function blocks.
191 *
192 * Important note: the caller must take whatever action is necessary to prevent
193 * any other updates to the existing list. In principle, it is possible to
194 * modify the list as soon as sync() begins execution. If this sort of thing
195 * becomes necessary, an alternative version based on call_rcu() could be
196 * created. But only if -really- needed -- there is no shortage of RCU API
197 * members.
198 */
199static inline void __list_splice_init_rcu(struct list_head *list,
200 struct list_head *prev,
201 struct list_head *next,
202 void (*sync)(void))
203{
204 struct list_head *first = list->next;
205 struct list_head *last = list->prev;
206
207 /*
208 * "first" and "last" tracking list, so initialize it. RCU readers
209 * have access to this list, so we must use INIT_LIST_HEAD_RCU()
210 * instead of INIT_LIST_HEAD().
211 */
212
213 INIT_LIST_HEAD_RCU(list);
214
215 /*
216 * At this point, the list body still points to the source list.
217 * Wait for any readers to finish using the list before splicing
218 * the list body into the new list. Any new readers will see
219 * an empty list.
220 */
221
222 sync();
223
224 /*
225 * Readers are finished with the source list, so perform splice.
226 * The order is important if the new list is global and accessible
227 * to concurrent RCU readers. Note that RCU readers are not
228 * permitted to traverse the prev pointers without excluding
229 * this function.
230 */
231
232 last->next = next;
233 rcu_assign_pointer(list_next_rcu(prev), first);
234 first->prev = prev;
235 next->prev = last;
236}
237
238/**
239 * list_splice_init_rcu - splice an RCU-protected list into an existing list,
240 * designed for stacks.
241 * @list: the RCU-protected list to splice
242 * @head: the place in the existing list to splice the first list into
243 * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
244 */
245static inline void list_splice_init_rcu(struct list_head *list,
246 struct list_head *head,
247 void (*sync)(void))
248{
249 if (!list_empty(list))
250 __list_splice_init_rcu(list, head, head->next, sync);
251}
252
253/**
254 * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
255 * list, designed for queues.
256 * @list: the RCU-protected list to splice
257 * @head: the place in the existing list to splice the first list into
258 * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
259 */
260static inline void list_splice_tail_init_rcu(struct list_head *list,
261 struct list_head *head,
262 void (*sync)(void))
263{
264 if (!list_empty(list))
265 __list_splice_init_rcu(list, head->prev, head, sync);
266}
267
268/**
269 * list_entry_rcu - get the struct for this entry
270 * @ptr: the &struct list_head pointer.
271 * @type: the type of the struct this is embedded in.
272 * @member: the name of the list_head within the struct.
273 *
274 * This primitive may safely run concurrently with the _rcu list-mutation
275 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
276 */
277#define list_entry_rcu(ptr, type, member) \
278 container_of(READ_ONCE(ptr), type, member)
279
280/*
281 * Where are list_empty_rcu() and list_first_entry_rcu()?
282 *
283 * Implementing those functions following their counterparts list_empty() and
284 * list_first_entry() is not advisable because they lead to subtle race
285 * conditions as the following snippet shows:
286 *
287 * if (!list_empty_rcu(mylist)) {
288 * struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
289 * do_something(bar);
290 * }
291 *
292 * The list may not be empty when list_empty_rcu checks it, but it may be when
293 * list_first_entry_rcu rereads the ->next pointer.
294 *
295 * Rereading the ->next pointer is not a problem for list_empty() and
296 * list_first_entry() because they would be protected by a lock that blocks
297 * writers.
298 *
299 * See list_first_or_null_rcu for an alternative.
300 */
301
302/**
303 * list_first_or_null_rcu - get the first element from a list
304 * @ptr: the list head to take the element from.
305 * @type: the type of the struct this is embedded in.
306 * @member: the name of the list_head within the struct.
307 *
308 * Note that if the list is empty, it returns NULL.
309 *
310 * This primitive may safely run concurrently with the _rcu list-mutation
311 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
312 */
313#define list_first_or_null_rcu(ptr, type, member) \
314({ \
315 struct list_head *__ptr = (ptr); \
316 struct list_head *__next = READ_ONCE(__ptr->next); \
317 likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
318})
319
320/**
321 * list_next_or_null_rcu - get the first element from a list
322 * @head: the head for the list.
323 * @ptr: the list head to take the next element from.
324 * @type: the type of the struct this is embedded in.
325 * @member: the name of the list_head within the struct.
326 *
327 * Note that if the ptr is at the end of the list, NULL is returned.
328 *
329 * This primitive may safely run concurrently with the _rcu list-mutation
330 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
331 */
332#define list_next_or_null_rcu(head, ptr, type, member) \
333({ \
334 struct list_head *__head = (head); \
335 struct list_head *__ptr = (ptr); \
336 struct list_head *__next = READ_ONCE(__ptr->next); \
337 likely(__next != __head) ? list_entry_rcu(__next, type, \
338 member) : NULL; \
339})
340
341/**
342 * list_for_each_entry_rcu - iterate over rcu list of given type
343 * @pos: the type * to use as a loop cursor.
344 * @head: the head for your list.
345 * @member: the name of the list_head within the struct.
346 *
347 * This list-traversal primitive may safely run concurrently with
348 * the _rcu list-mutation primitives such as list_add_rcu()
349 * as long as the traversal is guarded by rcu_read_lock().
350 */
351#define list_for_each_entry_rcu(pos, head, member) \
352 for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
353 &pos->member != (head); \
354 pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
355
356/**
357 * list_entry_lockless - get the struct for this entry
358 * @ptr: the &struct list_head pointer.
359 * @type: the type of the struct this is embedded in.
360 * @member: the name of the list_head within the struct.
361 *
362 * This primitive may safely run concurrently with the _rcu list-mutation
363 * primitives such as list_add_rcu(), but requires some implicit RCU
364 * read-side guarding. One example is running within a special
365 * exception-time environment where preemption is disabled and where
366 * lockdep cannot be invoked (in which case updaters must use RCU-sched,
367 * as in synchronize_sched(), call_rcu_sched(), and friends). Another
368 * example is when items are added to the list, but never deleted.
369 */
370#define list_entry_lockless(ptr, type, member) \
371 container_of((typeof(ptr))READ_ONCE(ptr), type, member)
372
373/**
374 * list_for_each_entry_lockless - iterate over rcu list of given type
375 * @pos: the type * to use as a loop cursor.
376 * @head: the head for your list.
377 * @member: the name of the list_struct within the struct.
378 *
379 * This primitive may safely run concurrently with the _rcu list-mutation
380 * primitives such as list_add_rcu(), but requires some implicit RCU
381 * read-side guarding. One example is running within a special
382 * exception-time environment where preemption is disabled and where
383 * lockdep cannot be invoked (in which case updaters must use RCU-sched,
384 * as in synchronize_sched(), call_rcu_sched(), and friends). Another
385 * example is when items are added to the list, but never deleted.
386 */
387#define list_for_each_entry_lockless(pos, head, member) \
388 for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
389 &pos->member != (head); \
390 pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
391
392/**
393 * list_for_each_entry_continue_rcu - continue iteration over list of given type
394 * @pos: the type * to use as a loop cursor.
395 * @head: the head for your list.
396 * @member: the name of the list_head within the struct.
397 *
398 * Continue to iterate over list of given type, continuing after
399 * the current position which must have been in the list when the RCU read
400 * lock was taken.
401 * This would typically require either that you obtained the node from a
402 * previous walk of the list in the same RCU read-side critical section, or
403 * that you held some sort of non-RCU reference (such as a reference count)
404 * to keep the node alive *and* in the list.
405 *
406 * This iterator is similar to list_for_each_entry_from_rcu() except
407 * this starts after the given position and that one starts at the given
408 * position.
409 */
410#define list_for_each_entry_continue_rcu(pos, head, member) \
411 for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
412 &pos->member != (head); \
413 pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
414
415/**
416 * list_for_each_entry_from_rcu - iterate over a list from current point
417 * @pos: the type * to use as a loop cursor.
418 * @head: the head for your list.
419 * @member: the name of the list_node within the struct.
420 *
421 * Iterate over the tail of a list starting from a given position,
422 * which must have been in the list when the RCU read lock was taken.
423 * This would typically require either that you obtained the node from a
424 * previous walk of the list in the same RCU read-side critical section, or
425 * that you held some sort of non-RCU reference (such as a reference count)
426 * to keep the node alive *and* in the list.
427 *
428 * This iterator is similar to list_for_each_entry_continue_rcu() except
429 * this starts from the given position and that one starts from the position
430 * after the given position.
431 */
432#define list_for_each_entry_from_rcu(pos, head, member) \
433 for (; &(pos)->member != (head); \
434 pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
435
436/**
437 * hlist_del_rcu - deletes entry from hash list without re-initialization
438 * @n: the element to delete from the hash list.
439 *
440 * Note: list_unhashed() on entry does not return true after this,
441 * the entry is in an undefined state. It is useful for RCU based
442 * lockfree traversal.
443 *
444 * In particular, it means that we can not poison the forward
445 * pointers that may still be used for walking the hash list.
446 *
447 * The caller must take whatever precautions are necessary
448 * (such as holding appropriate locks) to avoid racing
449 * with another list-mutation primitive, such as hlist_add_head_rcu()
450 * or hlist_del_rcu(), running on this same list.
451 * However, it is perfectly legal to run concurrently with
452 * the _rcu list-traversal primitives, such as
453 * hlist_for_each_entry().
454 */
455static inline void hlist_del_rcu(struct hlist_node *n)
456{
457 __hlist_del(n);
458 n->pprev = LIST_POISON2;
459}
460
461/**
462 * hlist_replace_rcu - replace old entry by new one
463 * @old : the element to be replaced
464 * @new : the new element to insert
465 *
466 * The @old entry will be replaced with the @new entry atomically.
467 */
468static inline void hlist_replace_rcu(struct hlist_node *old,
469 struct hlist_node *new)
470{
471 struct hlist_node *next = old->next;
472
473 new->next = next;
474 new->pprev = old->pprev;
475 rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
476 if (next)
477 new->next->pprev = &new->next;
478 old->pprev = LIST_POISON2;
479}
480
481/*
482 * return the first or the next element in an RCU protected hlist
483 */
484#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
485#define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
486#define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
487
488/**
489 * hlist_add_head_rcu
490 * @n: the element to add to the hash list.
491 * @h: the list to add to.
492 *
493 * Description:
494 * Adds the specified element to the specified hlist,
495 * while permitting racing traversals.
496 *
497 * The caller must take whatever precautions are necessary
498 * (such as holding appropriate locks) to avoid racing
499 * with another list-mutation primitive, such as hlist_add_head_rcu()
500 * or hlist_del_rcu(), running on this same list.
501 * However, it is perfectly legal to run concurrently with
502 * the _rcu list-traversal primitives, such as
503 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
504 * problems on Alpha CPUs. Regardless of the type of CPU, the
505 * list-traversal primitive must be guarded by rcu_read_lock().
506 */
507static inline void hlist_add_head_rcu(struct hlist_node *n,
508 struct hlist_head *h)
509{
510 struct hlist_node *first = h->first;
511
512 n->next = first;
513 n->pprev = &h->first;
514 rcu_assign_pointer(hlist_first_rcu(h), n);
515 if (first)
516 first->pprev = &n->next;
517}
518
519/**
520 * hlist_add_tail_rcu
521 * @n: the element to add to the hash list.
522 * @h: the list to add to.
523 *
524 * Description:
525 * Adds the specified element to the specified hlist,
526 * while permitting racing traversals.
527 *
528 * The caller must take whatever precautions are necessary
529 * (such as holding appropriate locks) to avoid racing
530 * with another list-mutation primitive, such as hlist_add_head_rcu()
531 * or hlist_del_rcu(), running on this same list.
532 * However, it is perfectly legal to run concurrently with
533 * the _rcu list-traversal primitives, such as
534 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
535 * problems on Alpha CPUs. Regardless of the type of CPU, the
536 * list-traversal primitive must be guarded by rcu_read_lock().
537 */
538static inline void hlist_add_tail_rcu(struct hlist_node *n,
539 struct hlist_head *h)
540{
541 struct hlist_node *i, *last = NULL;
542
543 /* Note: write side code, so rcu accessors are not needed. */
544 for (i = h->first; i; i = i->next)
545 last = i;
546
547 if (last) {
548 n->next = last->next;
549 n->pprev = &last->next;
550 rcu_assign_pointer(hlist_next_rcu(last), n);
551 } else {
552 hlist_add_head_rcu(n, h);
553 }
554}
555
556/**
557 * hlist_add_before_rcu
558 * @n: the new element to add to the hash list.
559 * @next: the existing element to add the new element before.
560 *
561 * Description:
562 * Adds the specified element to the specified hlist
563 * before the specified node while permitting racing traversals.
564 *
565 * The caller must take whatever precautions are necessary
566 * (such as holding appropriate locks) to avoid racing
567 * with another list-mutation primitive, such as hlist_add_head_rcu()
568 * or hlist_del_rcu(), running on this same list.
569 * However, it is perfectly legal to run concurrently with
570 * the _rcu list-traversal primitives, such as
571 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
572 * problems on Alpha CPUs.
573 */
574static inline void hlist_add_before_rcu(struct hlist_node *n,
575 struct hlist_node *next)
576{
577 n->pprev = next->pprev;
578 n->next = next;
579 rcu_assign_pointer(hlist_pprev_rcu(n), n);
580 next->pprev = &n->next;
581}
582
583/**
584 * hlist_add_behind_rcu
585 * @n: the new element to add to the hash list.
586 * @prev: the existing element to add the new element after.
587 *
588 * Description:
589 * Adds the specified element to the specified hlist
590 * after the specified node while permitting racing traversals.
591 *
592 * The caller must take whatever precautions are necessary
593 * (such as holding appropriate locks) to avoid racing
594 * with another list-mutation primitive, such as hlist_add_head_rcu()
595 * or hlist_del_rcu(), running on this same list.
596 * However, it is perfectly legal to run concurrently with
597 * the _rcu list-traversal primitives, such as
598 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
599 * problems on Alpha CPUs.
600 */
601static inline void hlist_add_behind_rcu(struct hlist_node *n,
602 struct hlist_node *prev)
603{
604 n->next = prev->next;
605 n->pprev = &prev->next;
606 rcu_assign_pointer(hlist_next_rcu(prev), n);
607 if (n->next)
608 n->next->pprev = &n->next;
609}
610
611#define __hlist_for_each_rcu(pos, head) \
612 for (pos = rcu_dereference(hlist_first_rcu(head)); \
613 pos; \
614 pos = rcu_dereference(hlist_next_rcu(pos)))
615
616/**
617 * hlist_for_each_entry_rcu - iterate over rcu list of given type
618 * @pos: the type * to use as a loop cursor.
619 * @head: the head for your list.
620 * @member: the name of the hlist_node within the struct.
621 *
622 * This list-traversal primitive may safely run concurrently with
623 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
624 * as long as the traversal is guarded by rcu_read_lock().
625 */
626#define hlist_for_each_entry_rcu(pos, head, member) \
627 for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\
628 typeof(*(pos)), member); \
629 pos; \
630 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
631 &(pos)->member)), typeof(*(pos)), member))
632
633/**
634 * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
635 * @pos: the type * to use as a loop cursor.
636 * @head: the head for your list.
637 * @member: the name of the hlist_node within the struct.
638 *
639 * This list-traversal primitive may safely run concurrently with
640 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
641 * as long as the traversal is guarded by rcu_read_lock().
642 *
643 * This is the same as hlist_for_each_entry_rcu() except that it does
644 * not do any RCU debugging or tracing.
645 */
646#define hlist_for_each_entry_rcu_notrace(pos, head, member) \
647 for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\
648 typeof(*(pos)), member); \
649 pos; \
650 pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\
651 &(pos)->member)), typeof(*(pos)), member))
652
653/**
654 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
655 * @pos: the type * to use as a loop cursor.
656 * @head: the head for your list.
657 * @member: the name of the hlist_node within the struct.
658 *
659 * This list-traversal primitive may safely run concurrently with
660 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
661 * as long as the traversal is guarded by rcu_read_lock().
662 */
663#define hlist_for_each_entry_rcu_bh(pos, head, member) \
664 for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_first_rcu(head)),\
665 typeof(*(pos)), member); \
666 pos; \
667 pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(\
668 &(pos)->member)), typeof(*(pos)), member))
669
670/**
671 * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
672 * @pos: the type * to use as a loop cursor.
673 * @member: the name of the hlist_node within the struct.
674 */
675#define hlist_for_each_entry_continue_rcu(pos, member) \
676 for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
677 &(pos)->member)), typeof(*(pos)), member); \
678 pos; \
679 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
680 &(pos)->member)), typeof(*(pos)), member))
681
682/**
683 * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
684 * @pos: the type * to use as a loop cursor.
685 * @member: the name of the hlist_node within the struct.
686 */
687#define hlist_for_each_entry_continue_rcu_bh(pos, member) \
688 for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
689 &(pos)->member)), typeof(*(pos)), member); \
690 pos; \
691 pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
692 &(pos)->member)), typeof(*(pos)), member))
693
694/**
695 * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
696 * @pos: the type * to use as a loop cursor.
697 * @member: the name of the hlist_node within the struct.
698 */
699#define hlist_for_each_entry_from_rcu(pos, member) \
700 for (; pos; \
701 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
702 &(pos)->member)), typeof(*(pos)), member))
703
704#endif /* __KERNEL__ */
705#endif
706