summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2005-10-17 05:29:20 +0200
committerLinus Torvalds <torvalds@g5.osdl.org>2005-10-17 17:59:10 +0200
commitb24d18aa743dad0c42918157c5d717686269d3a8 (patch)
treeceade1641a1806c5f3db2f4b344fe9070b488bad
parentFix memory ordering bug in page reclaim (diff)
downloadlinux-b24d18aa743dad0c42918157c5d717686269d3a8.tar.xz
linux-b24d18aa743dad0c42918157c5d717686269d3a8.zip
[PATCH] list: add missing rcu_dereference on first element
It seems that all the list_*_rcu primitives are missing a memory barrier on the very first dereference. For example, #define list_for_each_rcu(pos, head) \ for (pos = (head)->next; prefetch(pos->next), pos != (head); \ pos = rcu_dereference(pos->next)) It will go something like: pos = (head)->next prefetch(pos->next) pos != (head) do stuff We're missing a barrier here. pos = rcu_dereference(pos->next) fetch pos->next barrier given by rcu_dereference(pos->next) store pos Without the missing barrier, the pos->next value may turn out to be stale. In fact, if "do stuff" were also dereferencing pos and relying on list_for_each_rcu to provide the barrier then it may also break. So here is a patch to make sure that we have a barrier for the first element in the list. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Acked-by: "Paul E. McKenney" <paulmck@us.ibm.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r--include/linux/list.h39
1 files changed, 22 insertions, 17 deletions
diff --git a/include/linux/list.h b/include/linux/list.h
index e6ec59682274..084971f333fe 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -442,12 +442,14 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock().
*/
#define list_for_each_rcu(pos, head) \
- for (pos = (head)->next; prefetch(pos->next), pos != (head); \
- pos = rcu_dereference(pos->next))
+ for (pos = (head)->next; \
+ prefetch(rcu_dereference(pos)->next), pos != (head); \
+ pos = pos->next)
#define __list_for_each_rcu(pos, head) \
- for (pos = (head)->next; pos != (head); \
- pos = rcu_dereference(pos->next))
+ for (pos = (head)->next; \
+ rcu_dereference(pos) != (head); \
+ pos = pos->next)
/**
* list_for_each_safe_rcu - iterate over an rcu-protected list safe
@@ -461,8 +463,9 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock().
*/
#define list_for_each_safe_rcu(pos, n, head) \
- for (pos = (head)->next, n = pos->next; pos != (head); \
- pos = rcu_dereference(n), n = pos->next)
+ for (pos = (head)->next; \
+ n = rcu_dereference(pos)->next, pos != (head); \
+ pos = n)
/**
* list_for_each_entry_rcu - iterate over rcu list of given type
@@ -474,11 +477,11 @@ static inline void list_splice_init(struct list_head *list,
* the _rcu list-mutation primitives such as list_add_rcu()
* as long as the traversal is guarded by rcu_read_lock().
*/
-#define list_for_each_entry_rcu(pos, head, member) \
- for (pos = list_entry((head)->next, typeof(*pos), member); \
- prefetch(pos->member.next), &pos->member != (head); \
- pos = rcu_dereference(list_entry(pos->member.next, \
- typeof(*pos), member)))
+#define list_for_each_entry_rcu(pos, head, member) \
+ for (pos = list_entry((head)->next, typeof(*pos), member); \
+ prefetch(rcu_dereference(pos)->member.next), \
+ &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
/**
@@ -492,8 +495,9 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock().
*/
#define list_for_each_continue_rcu(pos, head) \
- for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
- (pos) = rcu_dereference((pos)->next))
+ for ((pos) = (pos)->next; \
+ prefetch(rcu_dereference((pos))->next), (pos) != (head); \
+ (pos) = (pos)->next)
/*
* Double linked lists with a single pointer list head.
@@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
pos = n)
#define hlist_for_each_rcu(pos, head) \
- for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
- (pos) = rcu_dereference((pos)->next))
+ for ((pos) = (head)->first; \
+ rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
+ (pos) = (pos)->next)
/**
* hlist_for_each_entry - iterate over list of given type
@@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
*/
#define hlist_for_each_entry_rcu(tpos, pos, head, member) \
for (pos = (head)->first; \
- pos && ({ prefetch(pos->next); 1;}) && \
+ rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
- pos = rcu_dereference(pos->next))
+ pos = pos->next)
#else
#warning "don't include kernel headers in userspace"