|
@@ -388,174 +388,92 @@ enum stopmachine_state {
|
|
|
/* Exit */
|
|
|
STOPMACHINE_EXIT,
|
|
|
};
|
|
|
-static enum stopmachine_state state;
|
|
|
|
|
|
struct stop_machine_data {
|
|
|
- int (*fn)(void *);
|
|
|
- void *data;
|
|
|
- int fnret;
|
|
|
+ int (*fn)(void *);
|
|
|
+ void *data;
|
|
|
+ /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
|
|
|
+ unsigned int num_threads;
|
|
|
+ const struct cpumask *active_cpus;
|
|
|
+
|
|
|
+ enum stopmachine_state state;
|
|
|
+ atomic_t thread_ack;
|
|
|
};
|
|
|
|
|
|
-/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
|
|
|
-static unsigned int num_threads;
|
|
|
-static atomic_t thread_ack;
|
|
|
-static DEFINE_MUTEX(lock);
|
|
|
-/* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
|
|
|
-static DEFINE_MUTEX(setup_lock);
|
|
|
-/* Users of stop_machine. */
|
|
|
-static int refcount;
|
|
|
-static struct workqueue_struct *stop_machine_wq;
|
|
|
-static struct stop_machine_data active, idle;
|
|
|
-static const struct cpumask *active_cpus;
|
|
|
-static void __percpu *stop_machine_work;
|
|
|
-
|
|
|
-static void set_state(enum stopmachine_state newstate)
|
|
|
+static void set_state(struct stop_machine_data *smdata,
|
|
|
+ enum stopmachine_state newstate)
|
|
|
{
|
|
|
/* Reset ack counter. */
|
|
|
- atomic_set(&thread_ack, num_threads);
|
|
|
+ atomic_set(&smdata->thread_ack, smdata->num_threads);
|
|
|
smp_wmb();
|
|
|
- state = newstate;
|
|
|
+ smdata->state = newstate;
|
|
|
}
|
|
|
|
|
|
/* Last one to ack a state moves to the next state. */
|
|
|
-static void ack_state(void)
|
|
|
+static void ack_state(struct stop_machine_data *smdata)
|
|
|
{
|
|
|
- if (atomic_dec_and_test(&thread_ack))
|
|
|
- set_state(state + 1);
|
|
|
+ if (atomic_dec_and_test(&smdata->thread_ack))
|
|
|
+ set_state(smdata, smdata->state + 1);
|
|
|
}
|
|
|
|
|
|
-/* This is the actual function which stops the CPU. It runs
|
|
|
- * in the context of a dedicated stopmachine workqueue. */
|
|
|
-static void stop_cpu(struct work_struct *unused)
|
|
|
+/* This is the cpu_stop function which stops the CPU. */
|
|
|
+static int stop_machine_cpu_stop(void *data)
|
|
|
{
|
|
|
+ struct stop_machine_data *smdata = data;
|
|
|
enum stopmachine_state curstate = STOPMACHINE_NONE;
|
|
|
- struct stop_machine_data *smdata = &idle;
|
|
|
- int cpu = smp_processor_id();
|
|
|
- int err;
|
|
|
+ int cpu = smp_processor_id(), err = 0;
|
|
|
+ bool is_active;
|
|
|
+
|
|
|
+ if (!smdata->active_cpus)
|
|
|
+ is_active = cpu == cpumask_first(cpu_online_mask);
|
|
|
+ else
|
|
|
+ is_active = cpumask_test_cpu(cpu, smdata->active_cpus);
|
|
|
|
|
|
- if (!active_cpus) {
|
|
|
- if (cpu == cpumask_first(cpu_online_mask))
|
|
|
- smdata = &active;
|
|
|
- } else {
|
|
|
- if (cpumask_test_cpu(cpu, active_cpus))
|
|
|
- smdata = &active;
|
|
|
- }
|
|
|
/* Simple state machine */
|
|
|
do {
|
|
|
/* Chill out and ensure we re-read stopmachine_state. */
|
|
|
cpu_relax();
|
|
|
- if (state != curstate) {
|
|
|
- curstate = state;
|
|
|
+ if (smdata->state != curstate) {
|
|
|
+ curstate = smdata->state;
|
|
|
switch (curstate) {
|
|
|
case STOPMACHINE_DISABLE_IRQ:
|
|
|
local_irq_disable();
|
|
|
hard_irq_disable();
|
|
|
break;
|
|
|
case STOPMACHINE_RUN:
|
|
|
- /* On multiple CPUs only a single error code
|
|
|
- * is needed to tell that something failed. */
|
|
|
- err = smdata->fn(smdata->data);
|
|
|
- if (err)
|
|
|
- smdata->fnret = err;
|
|
|
+ if (is_active)
|
|
|
+ err = smdata->fn(smdata->data);
|
|
|
break;
|
|
|
default:
|
|
|
break;
|
|
|
}
|
|
|
- ack_state();
|
|
|
+ ack_state(smdata);
|
|
|
}
|
|
|
} while (curstate != STOPMACHINE_EXIT);
|
|
|
|
|
|
local_irq_enable();
|
|
|
+ return err;
|
|
|
}
|
|
|
|
|
|
-/* Callback for CPUs which aren't supposed to do anything. */
|
|
|
-static int chill(void *unused)
|
|
|
-{
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-int stop_machine_create(void)
|
|
|
-{
|
|
|
- mutex_lock(&setup_lock);
|
|
|
- if (refcount)
|
|
|
- goto done;
|
|
|
- stop_machine_wq = create_rt_workqueue("kstop");
|
|
|
- if (!stop_machine_wq)
|
|
|
- goto err_out;
|
|
|
- stop_machine_work = alloc_percpu(struct work_struct);
|
|
|
- if (!stop_machine_work)
|
|
|
- goto err_out;
|
|
|
-done:
|
|
|
- refcount++;
|
|
|
- mutex_unlock(&setup_lock);
|
|
|
- return 0;
|
|
|
-
|
|
|
-err_out:
|
|
|
- if (stop_machine_wq)
|
|
|
- destroy_workqueue(stop_machine_wq);
|
|
|
- mutex_unlock(&setup_lock);
|
|
|
- return -ENOMEM;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(stop_machine_create);
|
|
|
-
|
|
|
-void stop_machine_destroy(void)
|
|
|
-{
|
|
|
- mutex_lock(&setup_lock);
|
|
|
- refcount--;
|
|
|
- if (refcount)
|
|
|
- goto done;
|
|
|
- destroy_workqueue(stop_machine_wq);
|
|
|
- free_percpu(stop_machine_work);
|
|
|
-done:
|
|
|
- mutex_unlock(&setup_lock);
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(stop_machine_destroy);
|
|
|
-
|
|
|
int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
|
|
|
{
|
|
|
- struct work_struct *sm_work;
|
|
|
- int i, ret;
|
|
|
-
|
|
|
- /* Set up initial state. */
|
|
|
- mutex_lock(&lock);
|
|
|
- num_threads = num_online_cpus();
|
|
|
- active_cpus = cpus;
|
|
|
- active.fn = fn;
|
|
|
- active.data = data;
|
|
|
- active.fnret = 0;
|
|
|
- idle.fn = chill;
|
|
|
- idle.data = NULL;
|
|
|
-
|
|
|
- set_state(STOPMACHINE_PREPARE);
|
|
|
-
|
|
|
- /* Schedule the stop_cpu work on all cpus: hold this CPU so one
|
|
|
- * doesn't hit this CPU until we're ready. */
|
|
|
- get_cpu();
|
|
|
- for_each_online_cpu(i) {
|
|
|
- sm_work = per_cpu_ptr(stop_machine_work, i);
|
|
|
- INIT_WORK(sm_work, stop_cpu);
|
|
|
- queue_work_on(i, stop_machine_wq, sm_work);
|
|
|
- }
|
|
|
- /* This will release the thread on our CPU. */
|
|
|
- put_cpu();
|
|
|
- flush_workqueue(stop_machine_wq);
|
|
|
- ret = active.fnret;
|
|
|
- mutex_unlock(&lock);
|
|
|
- return ret;
|
|
|
+ struct stop_machine_data smdata = { .fn = fn, .data = data,
|
|
|
+ .num_threads = num_online_cpus(),
|
|
|
+ .active_cpus = cpus };
|
|
|
+
|
|
|
+ /* Set the initial state and stop all online cpus. */
|
|
|
+ set_state(&smdata, STOPMACHINE_PREPARE);
|
|
|
+ return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
|
|
|
}
|
|
|
|
|
|
int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
|
|
|
{
|
|
|
int ret;
|
|
|
|
|
|
- ret = stop_machine_create();
|
|
|
- if (ret)
|
|
|
- return ret;
|
|
|
/* No CPUs can come up or down during this. */
|
|
|
get_online_cpus();
|
|
|
ret = __stop_machine(fn, data, cpus);
|
|
|
put_online_cpus();
|
|
|
- stop_machine_destroy();
|
|
|
return ret;
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(stop_machine);
|