1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_RCULIST_H
3#define _LINUX_RCULIST_H
4
5#ifdef __KERNEL__
6
7/*
8 * RCU-protected list version
9 */
10#include <linux/list.h>
11#include <linux/rcupdate.h>
12
13/*
14 * Why is there no list_empty_rcu()? Because list_empty() serves this
15 * purpose. The list_empty() function fetches the RCU-protected pointer
16 * and compares it to the address of the list head, but neither dereferences
17 * this pointer itself nor provides this pointer to the caller. Therefore,
18 * it is not necessary to use rcu_dereference(), so that list_empty() can
19 * be used anywhere you would want to use a list_empty_rcu().
20 */
21
22/*
23 * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
24 * @list: list to be initialized
25 *
26 * You should instead use INIT_LIST_HEAD() for normal initialization and
27 * cleanup tasks, when readers have no access to the list being initialized.
28 * However, if the list being initialized is visible to readers, you
29 * need to keep the compiler from being too mischievous.
30 */
31static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
32{
33 WRITE_ONCE(list->next, list);
34 WRITE_ONCE(list->prev, list);
35}
36
37/*
38 * return the ->next pointer of a list_head in an rcu safe
39 * way, we must not access it directly
40 */
41#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
42
43/*
44 * Insert a new entry between two known consecutive entries.
45 *
46 * This is only for internal list manipulation where we know
47 * the prev/next entries already!
48 */
49static inline void __list_add_rcu(struct list_head *new,
50 struct list_head *prev, struct list_head *next)
51{
52 if (!__list_add_valid(new, prev, next))
53 return;
54
55 new->next = next;
56 new->prev = prev;
57 rcu_assign_pointer(list_next_rcu(prev), new);
58 next->prev = new;
59}
60
61/**
62 * list_add_rcu - add a new entry to rcu-protected list
63 * @new: new entry to be added
64 * @head: list head to add it after
65 *
66 * Insert a new entry after the specified head.
67 * This is good for implementing stacks.
68 *
69 * The caller must take whatever precautions are necessary
70 * (such as holding appropriate locks) to avoid racing
71 * with another list-mutation primitive, such as list_add_rcu()
72 * or list_del_rcu(), running on this same list.
73 * However, it is perfectly legal to run concurrently with
74 * the _rcu list-traversal primitives, such as
75 * list_for_each_entry_rcu().
76 */
77static inline void list_add_rcu(struct list_head *new, struct list_head *head)
78{
79 __list_add_rcu(new, head, head->next);
80}
81
82/**
83 * list_add_tail_rcu - add a new entry to rcu-protected list
84 * @new: new entry to be added
85 * @head: list head to add it before
86 *
87 * Insert a new entry before the specified head.
88 * This is useful for implementing queues.
89 *
90 * The caller must take whatever precautions are necessary
91 * (such as holding appropriate locks) to avoid racing
92 * with another list-mutation primitive, such as list_add_tail_rcu()
93 * or list_del_rcu(), running on this same list.
94 * However, it is perfectly legal to run concurrently with
95 * the _rcu list-traversal primitives, such as
96 * list_for_each_entry_rcu().
97 */
98static inline void list_add_tail_rcu(struct list_head *new,
99 struct list_head *head)
100{
101 __list_add_rcu(new, head->prev, head);
102}
103
104/**
105 * list_del_rcu - deletes entry from list without re-initialization
106 * @entry: the element to delete from the list.
107 *
108 * Note: list_empty() on entry does not return true after this,
109 * the entry is in an undefined state. It is useful for RCU based
110 * lockfree traversal.
111 *
112 * In particular, it means that we can not poison the forward
113 * pointers that may still be used for walking the list.
114 *
115 * The caller must take whatever precautions are necessary
116 * (such as holding appropriate locks) to avoid racing
117 * with another list-mutation primitive, such as list_del_rcu()
118 * or list_add_rcu(), running on this same list.
119 * However, it is perfectly legal to run concurrently with
120 * the _rcu list-traversal primitives, such as
121 * list_for_each_entry_rcu().
122 *
123 * Note that the caller is not permitted to immediately free
124 * the newly deleted entry. Instead, either synchronize_rcu()
125 * or call_rcu() must be used to defer freeing until an RCU
126 * grace period has elapsed.
127 */
128static inline void list_del_rcu(struct list_head *entry)
129{
130 __list_del_entry(entry);
131 entry->prev = LIST_POISON2;
132}
133
134/**
135 * hlist_del_init_rcu - deletes entry from hash list with re-initialization
136 * @n: the element to delete from the hash list.
137 *
138 * Note: list_unhashed() on the node return true after this. It is
139 * useful for RCU based read lockfree traversal if the writer side
140 * must know if the list entry is still hashed or already unhashed.
141 *
142 * In particular, it means that we can not poison the forward pointers
143 * that may still be used for walking the hash list and we can only
144 * zero the pprev pointer so list_unhashed() will return true after
145 * this.
146 *
147 * The caller must take whatever precautions are necessary (such as
148 * holding appropriate locks) to avoid racing with another
149 * list-mutation primitive, such as hlist_add_head_rcu() or
150 * hlist_del_rcu(), running on this same list. However, it is
151 * perfectly legal to run concurrently with the _rcu list-traversal
152 * primitives, such as hlist_for_each_entry_rcu().
153 */
154static inline void hlist_del_init_rcu(struct hlist_node *n)
155{
156 if (!hlist_unhashed(n)) {
157 __hlist_del(n);
158 n->pprev = NULL;
159 }
160}
161
162/**
163 * list_replace_rcu - replace old entry by new one
164 * @old : the element to be replaced
165 * @new : the new element to insert
166 *
167 * The @old entry will be replaced with the @new entry atomically.
168 * Note: @old should not be empty.
169 */
170static inline void list_replace_rcu(struct list_head *old,
171 struct list_head *new)
172{
173 new->next = old->next;
174 new->prev = old->prev;
175 rcu_assign_pointer(list_next_rcu(new->prev), new);
176 new->next->prev = new;
177 old->prev = LIST_POISON2;
178}
179
180/**
181 * __list_splice_init_rcu - join an RCU-protected list into an existing list.
182 * @list: the RCU-protected list to splice
183 * @prev: points to the last element of the existing list
184 * @next: points to the first element of the existing list
185 * @sync: synchronize_rcu, synchronize_rcu_expedited, ...
186 *
187 * The list pointed to by @prev and @next can be RCU-read traversed
188 * concurrently with this function.
189 *
190 * Note that this function blocks.
191 *
192 * Important note: the caller must take whatever action is necessary to prevent
193 * any other updates to the existing list. In principle, it is possible to
194 * modify the list as soon as sync() begins execution. If this sort of thing
195 * becomes necessary, an alternative version based on call_rcu() could be
196 * created. But only if -really- needed -- there is no shortage of RCU API
197 * members.
198 */
199static inline void __list_splice_init_rcu(struct list_head *list,
200 struct list_head *prev,
201 struct list_head *next,
202 void (*sync)(void))
203{
204 struct list_head *first = list->next;
205 struct list_head *last = list->prev;
206
207 /*
208 * "first" and "last" tracking list, so initialize it. RCU readers
209 * have access to this list, so we must use INIT_LIST_HEAD_RCU()
210 * instead of INIT_LIST_HEAD().
211 */
212
213 INIT_LIST_HEAD_RCU(list);
214
215 /*
216 * At this point, the list body still points to the source list.
217 * Wait for any readers to finish using the list before splicing
218 * the list body into the new list. Any new readers will see
219 * an empty list.
220 */
221
222 sync();
223
224 /*
225 * Readers are finished with the source list, so perform splice.
226 * The order is important if the new list is global and accessible
227 * to concurrent RCU readers. Note that RCU readers are not
228 * permitted to traverse the prev pointers without excluding
229 * this function.
230 */
231
232 last->next = next;
233 rcu_assign_pointer(list_next_rcu(prev), first);
234 first->prev = prev;
235 next->prev = last;
236}
237
238/**
239 * list_splice_init_rcu - splice an RCU-protected list into an existing list,
240 * designed for stacks.
241 * @list: the RCU-protected list to splice
242 * @head: the place in the existing list to splice the first list into
243 * @sync: synchronize_rcu, synchronize_rcu_expedited, ...
244 */
245static inline void list_splice_init_rcu(struct list_head *list,
246 struct list_head *head,
247 void (*sync)(void))
248{
249 if (!list_empty(list))
250 __list_splice_init_rcu(list, head, head->next, sync);
251}
252
253/**
254 * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
255 * list, designed for queues.
256 * @list: the RCU-protected list to splice
257 * @head: the place in the existing list to splice the first list into
258 * @sync: synchronize_rcu, synchronize_rcu_expedited, ...
259 */
260static inline void list_splice_tail_init_rcu(struct list_head *list,
261 struct list_head *head,
262 void (*sync)(void))
263{
264 if (!list_empty(list))
265 __list_splice_init_rcu(list, head->prev, head, sync);
266}
267
268/**
269 * list_entry_rcu - get the struct for this entry
270 * @ptr: the &struct list_head pointer.
271 * @type: the type of the struct this is embedded in.
272 * @member: the name of the list_head within the struct.
273 *
274 * This primitive may safely run concurrently with the _rcu list-mutation
275 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
276 */
277#define list_entry_rcu(ptr, type, member) \
278 container_of(READ_ONCE(ptr), type, member)
279
280/*
281 * Where are list_empty_rcu() and list_first_entry_rcu()?
282 *
283 * Implementing those functions following their counterparts list_empty() and
284 * list_first_entry() is not advisable because they lead to subtle race
285 * conditions as the following snippet shows:
286 *
287 * if (!list_empty_rcu(mylist)) {
288 * struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
289 * do_something(bar);
290 * }
291 *
292 * The list may not be empty when list_empty_rcu checks it, but it may be when
293 * list_first_entry_rcu rereads the ->next pointer.
294 *
295 * Rereading the ->next pointer is not a problem for list_empty() and
296 * list_first_entry() because they would be protected by a lock that blocks
297 * writers.
298 *
299 * See list_first_or_null_rcu for an alternative.
300 */
301
302/**
303 * list_first_or_null_rcu - get the first element from a list
304 * @ptr: the list head to take the element from.
305 * @type: the type of the struct this is embedded in.
306 * @member: the name of the list_head within the struct.
307 *
308 * Note that if the list is empty, it returns NULL.
309 *
310 * This primitive may safely run concurrently with the _rcu list-mutation
311 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
312 */
313#define list_first_or_null_rcu(ptr, type, member) \
314({ \
315 struct list_head *__ptr = (ptr); \
316 struct list_head *__next = READ_ONCE(__ptr->next); \
317 likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
318})
319
320/**
321 * list_next_or_null_rcu - get the first element from a list
322 * @head: the head for the list.
323 * @ptr: the list head to take the next element from.
324 * @type: the type of the struct this is embedded in.
325 * @member: the name of the list_head within the struct.
326 *
327 * Note that if the ptr is at the end of the list, NULL is returned.
328 *
329 * This primitive may safely run concurrently with the _rcu list-mutation
330 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
331 */
332#define list_next_or_null_rcu(head, ptr, type, member) \
333({ \
334 struct list_head *__head = (head); \
335 struct list_head *__ptr = (ptr); \
336 struct list_head *__next = READ_ONCE(__ptr->next); \
337 likely(__next != __head) ? list_entry_rcu(__next, type, \
338 member) : NULL; \
339})
340
341/**
342 * list_for_each_entry_rcu - iterate over rcu list of given type
343 * @pos: the type * to use as a loop cursor.
344 * @head: the head for your list.
345 * @member: the name of the list_head within the struct.
346 *
347 * This list-traversal primitive may safely run concurrently with
348 * the _rcu list-mutation primitives such as list_add_rcu()
349 * as long as the traversal is guarded by rcu_read_lock().
350 */
351#define list_for_each_entry_rcu(pos, head, member) \
352 for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
353 &pos->member != (head); \
354 pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
355
356/**
357 * list_entry_lockless - get the struct for this entry
358 * @ptr: the &struct list_head pointer.
359 * @type: the type of the struct this is embedded in.
360 * @member: the name of the list_head within the struct.
361 *
362 * This primitive may safely run concurrently with the _rcu
363 * list-mutation primitives such as list_add_rcu(), but requires some
364 * implicit RCU read-side guarding. One example is running within a special
365 * exception-time environment where preemption is disabled and where lockdep
366 * cannot be invoked. Another example is when items are added to the list,
367 * but never deleted.
368 */
369#define list_entry_lockless(ptr, type, member) \
370 container_of((typeof(ptr))READ_ONCE(ptr), type, member)
371
372/**
373 * list_for_each_entry_lockless - iterate over rcu list of given type
374 * @pos: the type * to use as a loop cursor.
375 * @head: the head for your list.
376 * @member: the name of the list_struct within the struct.
377 *
378 * This primitive may safely run concurrently with the _rcu
379 * list-mutation primitives such as list_add_rcu(), but requires some
380 * implicit RCU read-side guarding. One example is running within a special
381 * exception-time environment where preemption is disabled and where lockdep
382 * cannot be invoked. Another example is when items are added to the list,
383 * but never deleted.
384 */
385#define list_for_each_entry_lockless(pos, head, member) \
386 for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
387 &pos->member != (head); \
388 pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
389
390/**
391 * list_for_each_entry_continue_rcu - continue iteration over list of given type
392 * @pos: the type * to use as a loop cursor.
393 * @head: the head for your list.
394 * @member: the name of the list_head within the struct.
395 *
396 * Continue to iterate over list of given type, continuing after
397 * the current position which must have been in the list when the RCU read
398 * lock was taken.
399 * This would typically require either that you obtained the node from a
400 * previous walk of the list in the same RCU read-side critical section, or
401 * that you held some sort of non-RCU reference (such as a reference count)
402 * to keep the node alive *and* in the list.
403 *
404 * This iterator is similar to list_for_each_entry_from_rcu() except
405 * this starts after the given position and that one starts at the given
406 * position.
407 */
408#define list_for_each_entry_continue_rcu(pos, head, member) \
409 for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
410 &pos->member != (head); \
411 pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
412
413/**
414 * list_for_each_entry_from_rcu - iterate over a list from current point
415 * @pos: the type * to use as a loop cursor.
416 * @head: the head for your list.
417 * @member: the name of the list_node within the struct.
418 *
419 * Iterate over the tail of a list starting from a given position,
420 * which must have been in the list when the RCU read lock was taken.
421 * This would typically require either that you obtained the node from a
422 * previous walk of the list in the same RCU read-side critical section, or
423 * that you held some sort of non-RCU reference (such as a reference count)
424 * to keep the node alive *and* in the list.
425 *
426 * This iterator is similar to list_for_each_entry_continue_rcu() except
427 * this starts from the given position and that one starts from the position
428 * after the given position.
429 */
430#define list_for_each_entry_from_rcu(pos, head, member) \
431 for (; &(pos)->member != (head); \
432 pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
433
434/**
435 * hlist_del_rcu - deletes entry from hash list without re-initialization
436 * @n: the element to delete from the hash list.
437 *
438 * Note: list_unhashed() on entry does not return true after this,
439 * the entry is in an undefined state. It is useful for RCU based
440 * lockfree traversal.
441 *
442 * In particular, it means that we can not poison the forward
443 * pointers that may still be used for walking the hash list.
444 *
445 * The caller must take whatever precautions are necessary
446 * (such as holding appropriate locks) to avoid racing
447 * with another list-mutation primitive, such as hlist_add_head_rcu()
448 * or hlist_del_rcu(), running on this same list.
449 * However, it is perfectly legal to run concurrently with
450 * the _rcu list-traversal primitives, such as
451 * hlist_for_each_entry().
452 */
453static inline void hlist_del_rcu(struct hlist_node *n)
454{
455 __hlist_del(n);
456 n->pprev = LIST_POISON2;
457}
458
459/**
460 * hlist_replace_rcu - replace old entry by new one
461 * @old : the element to be replaced
462 * @new : the new element to insert
463 *
464 * The @old entry will be replaced with the @new entry atomically.
465 */
466static inline void hlist_replace_rcu(struct hlist_node *old,
467 struct hlist_node *new)
468{
469 struct hlist_node *next = old->next;
470
471 new->next = next;
472 new->pprev = old->pprev;
473 rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
474 if (next)
475 new->next->pprev = &new->next;
476 old->pprev = LIST_POISON2;
477}
478
479/*
480 * return the first or the next element in an RCU protected hlist
481 */
482#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
483#define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
484#define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
485
486/**
487 * hlist_add_head_rcu
488 * @n: the element to add to the hash list.
489 * @h: the list to add to.
490 *
491 * Description:
492 * Adds the specified element to the specified hlist,
493 * while permitting racing traversals.
494 *
495 * The caller must take whatever precautions are necessary
496 * (such as holding appropriate locks) to avoid racing
497 * with another list-mutation primitive, such as hlist_add_head_rcu()
498 * or hlist_del_rcu(), running on this same list.
499 * However, it is perfectly legal to run concurrently with
500 * the _rcu list-traversal primitives, such as
501 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
502 * problems on Alpha CPUs. Regardless of the type of CPU, the
503 * list-traversal primitive must be guarded by rcu_read_lock().
504 */
505static inline void hlist_add_head_rcu(struct hlist_node *n,
506 struct hlist_head *h)
507{
508 struct hlist_node *first = h->first;
509
510 n->next = first;
511 n->pprev = &h->first;
512 rcu_assign_pointer(hlist_first_rcu(h), n);
513 if (first)
514 first->pprev = &n->next;
515}
516
517/**
518 * hlist_add_tail_rcu
519 * @n: the element to add to the hash list.
520 * @h: the list to add to.
521 *
522 * Description:
523 * Adds the specified element to the specified hlist,
524 * while permitting racing traversals.
525 *
526 * The caller must take whatever precautions are necessary
527 * (such as holding appropriate locks) to avoid racing
528 * with another list-mutation primitive, such as hlist_add_head_rcu()
529 * or hlist_del_rcu(), running on this same list.
530 * However, it is perfectly legal to run concurrently with
531 * the _rcu list-traversal primitives, such as
532 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
533 * problems on Alpha CPUs. Regardless of the type of CPU, the
534 * list-traversal primitive must be guarded by rcu_read_lock().
535 */
536static inline void hlist_add_tail_rcu(struct hlist_node *n,
537 struct hlist_head *h)
538{
539 struct hlist_node *i, *last = NULL;
540
541 /* Note: write side code, so rcu accessors are not needed. */
542 for (i = h->first; i; i = i->next)
543 last = i;
544
545 if (last) {
546 n->next = last->next;
547 n->pprev = &last->next;
548 rcu_assign_pointer(hlist_next_rcu(last), n);
549 } else {
550 hlist_add_head_rcu(n, h);
551 }
552}
553
554/**
555 * hlist_add_before_rcu
556 * @n: the new element to add to the hash list.
557 * @next: the existing element to add the new element before.
558 *
559 * Description:
560 * Adds the specified element to the specified hlist
561 * before the specified node while permitting racing traversals.
562 *
563 * The caller must take whatever precautions are necessary
564 * (such as holding appropriate locks) to avoid racing
565 * with another list-mutation primitive, such as hlist_add_head_rcu()
566 * or hlist_del_rcu(), running on this same list.
567 * However, it is perfectly legal to run concurrently with
568 * the _rcu list-traversal primitives, such as
569 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
570 * problems on Alpha CPUs.
571 */
572static inline void hlist_add_before_rcu(struct hlist_node *n,
573 struct hlist_node *next)
574{
575 n->pprev = next->pprev;
576 n->next = next;
577 rcu_assign_pointer(hlist_pprev_rcu(n), n);
578 next->pprev = &n->next;
579}
580
581/**
582 * hlist_add_behind_rcu
583 * @n: the new element to add to the hash list.
584 * @prev: the existing element to add the new element after.
585 *
586 * Description:
587 * Adds the specified element to the specified hlist
588 * after the specified node while permitting racing traversals.
589 *
590 * The caller must take whatever precautions are necessary
591 * (such as holding appropriate locks) to avoid racing
592 * with another list-mutation primitive, such as hlist_add_head_rcu()
593 * or hlist_del_rcu(), running on this same list.
594 * However, it is perfectly legal to run concurrently with
595 * the _rcu list-traversal primitives, such as
596 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
597 * problems on Alpha CPUs.
598 */
599static inline void hlist_add_behind_rcu(struct hlist_node *n,
600 struct hlist_node *prev)
601{
602 n->next = prev->next;
603 n->pprev = &prev->next;
604 rcu_assign_pointer(hlist_next_rcu(prev), n);
605 if (n->next)
606 n->next->pprev = &n->next;
607}
608
609#define __hlist_for_each_rcu(pos, head) \
610 for (pos = rcu_dereference(hlist_first_rcu(head)); \
611 pos; \
612 pos = rcu_dereference(hlist_next_rcu(pos)))
613
614/**
615 * hlist_for_each_entry_rcu - iterate over rcu list of given type
616 * @pos: the type * to use as a loop cursor.
617 * @head: the head for your list.
618 * @member: the name of the hlist_node within the struct.
619 *
620 * This list-traversal primitive may safely run concurrently with
621 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
622 * as long as the traversal is guarded by rcu_read_lock().
623 */
624#define hlist_for_each_entry_rcu(pos, head, member) \
625 for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\
626 typeof(*(pos)), member); \
627 pos; \
628 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
629 &(pos)->member)), typeof(*(pos)), member))
630
631/**
632 * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
633 * @pos: the type * to use as a loop cursor.
634 * @head: the head for your list.
635 * @member: the name of the hlist_node within the struct.
636 *
637 * This list-traversal primitive may safely run concurrently with
638 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
639 * as long as the traversal is guarded by rcu_read_lock().
640 *
641 * This is the same as hlist_for_each_entry_rcu() except that it does
642 * not do any RCU debugging or tracing.
643 */
644#define hlist_for_each_entry_rcu_notrace(pos, head, member) \
645 for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\
646 typeof(*(pos)), member); \
647 pos; \
648 pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\
649 &(pos)->member)), typeof(*(pos)), member))
650
651/**
652 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
653 * @pos: the type * to use as a loop cursor.
654 * @head: the head for your list.
655 * @member: the name of the hlist_node within the struct.
656 *
657 * This list-traversal primitive may safely run concurrently with
658 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
659 * as long as the traversal is guarded by rcu_read_lock().
660 */
661#define hlist_for_each_entry_rcu_bh(pos, head, member) \
662 for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_first_rcu(head)),\
663 typeof(*(pos)), member); \
664 pos; \
665 pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(\
666 &(pos)->member)), typeof(*(pos)), member))
667
668/**
669 * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
670 * @pos: the type * to use as a loop cursor.
671 * @member: the name of the hlist_node within the struct.
672 */
673#define hlist_for_each_entry_continue_rcu(pos, member) \
674 for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
675 &(pos)->member)), typeof(*(pos)), member); \
676 pos; \
677 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
678 &(pos)->member)), typeof(*(pos)), member))
679
680/**
681 * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
682 * @pos: the type * to use as a loop cursor.
683 * @member: the name of the hlist_node within the struct.
684 */
685#define hlist_for_each_entry_continue_rcu_bh(pos, member) \
686 for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
687 &(pos)->member)), typeof(*(pos)), member); \
688 pos; \
689 pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
690 &(pos)->member)), typeof(*(pos)), member))
691
692/**
693 * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
694 * @pos: the type * to use as a loop cursor.
695 * @member: the name of the hlist_node within the struct.
696 */
697#define hlist_for_each_entry_from_rcu(pos, member) \
698 for (; pos; \
699 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
700 &(pos)->member)), typeof(*(pos)), member))
701
702#endif /* __KERNEL__ */
703#endif
704