|
@@ -64,34 +64,42 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed);
|
|
|
* not in a quiescent state. There might be any number of tasks blocked
|
|
|
* while in an RCU read-side critical section.
|
|
|
*/
|
|
|
-static void rcu_preempt_qs_record(int cpu)
|
|
|
+static void rcu_preempt_qs(int cpu)
|
|
|
{
|
|
|
struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
|
|
|
- rdp->passed_quiesc = 1;
|
|
|
rdp->passed_quiesc_completed = rdp->completed;
|
|
|
+ barrier();
|
|
|
+ rdp->passed_quiesc = 1;
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * We have entered the scheduler or are between softirqs in ksoftirqd.
|
|
|
- * If we are in an RCU read-side critical section, we need to reflect
|
|
|
- * that in the state of the rcu_node structure corresponding to this CPU.
|
|
|
- * Caller must disable hardirqs.
|
|
|
+ * We have entered the scheduler, and the current task might soon be
|
|
|
+ * context-switched away from. If this task is in an RCU read-side
|
|
|
+ * critical section, we will no longer be able to rely on the CPU to
|
|
|
+ * record that fact, so we enqueue the task on the appropriate entry
|
|
|
+ * of the blocked_tasks[] array. The task will dequeue itself when
|
|
|
+ * it exits the outermost enclosing RCU read-side critical section.
|
|
|
+ * Therefore, the current grace period cannot be permitted to complete
|
|
|
+ * until the blocked_tasks[] entry indexed by the low-order bit of
|
|
|
+ * rnp->gpnum empties.
|
|
|
+ *
|
|
|
+ * Caller must disable preemption.
|
|
|
*/
|
|
|
-static void rcu_preempt_qs(int cpu)
|
|
|
+static void rcu_preempt_note_context_switch(int cpu)
|
|
|
{
|
|
|
struct task_struct *t = current;
|
|
|
+ unsigned long flags;
|
|
|
int phase;
|
|
|
struct rcu_data *rdp;
|
|
|
struct rcu_node *rnp;
|
|
|
|
|
|
if (t->rcu_read_lock_nesting &&
|
|
|
(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
|
|
|
- WARN_ON_ONCE(cpu != smp_processor_id());
|
|
|
|
|
|
/* Possibly blocking in an RCU read-side critical section. */
|
|
|
rdp = rcu_preempt_state.rda[cpu];
|
|
|
rnp = rdp->mynode;
|
|
|
- spin_lock(&rnp->lock);
|
|
|
+ spin_lock_irqsave(&rnp->lock, flags);
|
|
|
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
|
|
|
t->rcu_blocked_node = rnp;
|
|
|
|
|
@@ -112,7 +120,7 @@ static void rcu_preempt_qs(int cpu)
|
|
|
phase = !(rnp->qsmask & rdp->grpmask) ^ (rnp->gpnum & 0x1);
|
|
|
list_add(&t->rcu_node_entry, &rnp->blocked_tasks[phase]);
|
|
|
smp_mb(); /* Ensure later ctxt swtch seen after above. */
|
|
|
- spin_unlock(&rnp->lock);
|
|
|
+ spin_unlock_irqrestore(&rnp->lock, flags);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -124,9 +132,8 @@ static void rcu_preempt_qs(int cpu)
|
|
|
* grace period, then the fact that the task has been enqueued
|
|
|
* means that we continue to block the current grace period.
|
|
|
*/
|
|
|
- rcu_preempt_qs_record(cpu);
|
|
|
- t->rcu_read_unlock_special &= ~(RCU_READ_UNLOCK_NEED_QS |
|
|
|
- RCU_READ_UNLOCK_GOT_QS);
|
|
|
+ rcu_preempt_qs(cpu);
|
|
|
+ t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -162,7 +169,7 @@ static void rcu_read_unlock_special(struct task_struct *t)
|
|
|
special = t->rcu_read_unlock_special;
|
|
|
if (special & RCU_READ_UNLOCK_NEED_QS) {
|
|
|
t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
|
|
|
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_GOT_QS;
|
|
|
+ rcu_preempt_qs(smp_processor_id());
|
|
|
}
|
|
|
|
|
|
/* Hardware IRQ handlers cannot block. */
|
|
@@ -199,9 +206,7 @@ static void rcu_read_unlock_special(struct task_struct *t)
|
|
|
*/
|
|
|
if (!empty && rnp->qsmask == 0 &&
|
|
|
list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1])) {
|
|
|
- t->rcu_read_unlock_special &=
|
|
|
- ~(RCU_READ_UNLOCK_NEED_QS |
|
|
|
- RCU_READ_UNLOCK_GOT_QS);
|
|
|
+ t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
|
|
|
if (rnp->parent == NULL) {
|
|
|
/* Only one rcu_node in the tree. */
|
|
|
cpu_quiet_msk_finish(&rcu_preempt_state, flags);
|
|
@@ -352,19 +357,12 @@ static void rcu_preempt_check_callbacks(int cpu)
|
|
|
struct task_struct *t = current;
|
|
|
|
|
|
if (t->rcu_read_lock_nesting == 0) {
|
|
|
- t->rcu_read_unlock_special &=
|
|
|
- ~(RCU_READ_UNLOCK_NEED_QS | RCU_READ_UNLOCK_GOT_QS);
|
|
|
- rcu_preempt_qs_record(cpu);
|
|
|
+ t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
|
|
|
+ rcu_preempt_qs(cpu);
|
|
|
return;
|
|
|
}
|
|
|
if (per_cpu(rcu_preempt_data, cpu).qs_pending) {
|
|
|
- if (t->rcu_read_unlock_special & RCU_READ_UNLOCK_GOT_QS) {
|
|
|
- rcu_preempt_qs_record(cpu);
|
|
|
- t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_GOT_QS;
|
|
|
- } else if (!(t->rcu_read_unlock_special &
|
|
|
- RCU_READ_UNLOCK_NEED_QS)) {
|
|
|
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
|
|
|
- }
|
|
|
+ t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -451,7 +449,7 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed);
|
|
|
* Because preemptable RCU does not exist, we never have to check for
|
|
|
* CPUs being in quiescent states.
|
|
|
*/
|
|
|
-static void rcu_preempt_qs(int cpu)
|
|
|
+static void rcu_preempt_note_context_switch(int cpu)
|
|
|
{
|
|
|
}
|
|
|
|