diff options
| author | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2016-11-28 21:08:49 +0100 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2017-01-25 21:54:22 +0100 |
| commit | d85b62f18d543c663cbdd6061054efeb9e66cee7 (patch) | |
| tree | f057b276302012d5f62a6c0a2727e0b334e26866 /kernel/rcu/srcu.c | |
| parent | srcu: Implement more-efficient reader counts (diff) | |
| download | linux-d85b62f18d543c663cbdd6061054efeb9e66cee7.tar.xz linux-d85b62f18d543c663cbdd6061054efeb9e66cee7.zip | |
srcu: Force full grace-period ordering
If a process invokes synchronize_srcu(), is delayed just the right amount
of time, and thus does not sleep when waiting for the grace period to
complete, there is no ordering between the end of the grace period and
the code following the synchronize_srcu(). Similarly, there can be a
lack of ordering between the end of the SRCU grace period and callback
invocation.
This commit adds the necessary ordering.
Reported-by: Lance Roy <ldr709@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Further smp_mb() adjustment per email with Lance Roy. ]
Diffstat (limited to 'kernel/rcu/srcu.c')
| -rw-r--r-- | kernel/rcu/srcu.c | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c index c9a0015e1c2e..665bc9951523 100644 --- a/kernel/rcu/srcu.c +++ b/kernel/rcu/srcu.c @@ -358,6 +358,7 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *head, head->next = NULL; head->func = func; spin_lock_irqsave(&sp->queue_lock, flags); + smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */ rcu_batch_queue(&sp->batch_queue, head); if (!sp->running) { sp->running = true; @@ -391,6 +392,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount) head->next = NULL; head->func = wakeme_after_rcu; spin_lock_irq(&sp->queue_lock); + smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */ if (!sp->running) { /* steal the processing owner */ sp->running = true; @@ -410,8 +412,11 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount) spin_unlock_irq(&sp->queue_lock); } - if (!done) + if (!done) { wait_for_completion(&rcu.completion); + smp_mb(); /* Caller's later accesses after GP. */ + } + } /** @@ -579,7 +584,8 @@ static void srcu_advance_batches(struct srcu_struct *sp, int trycount) /* * Invoke a limited number of SRCU callbacks that have passed through * their grace period. If there are more to do, SRCU will reschedule - * the workqueue. + * the workqueue. Note that needed memory barriers have been executed + * in this task's context by srcu_readers_active_idx_check(). */ static void srcu_invoke_callbacks(struct srcu_struct *sp) { |