smp.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688
  1. /*
  2. * Generic helpers for smp ipi calls
  3. *
  4. * (C) Jens Axboe <jens.axboe@oracle.com> 2008
  5. */
  6. #include <linux/rcupdate.h>
  7. #include <linux/rculist.h>
  8. #include <linux/kernel.h>
  9. #include <linux/export.h>
  10. #include <linux/percpu.h>
  11. #include <linux/init.h>
  12. #include <linux/gfp.h>
  13. #include <linux/smp.h>
  14. #include <linux/cpu.h>
  15. #include "smpboot.h"
  16. enum {
  17. CSD_FLAG_LOCK = 0x01,
  18. CSD_FLAG_WAIT = 0x02,
  19. };
  20. struct call_function_data {
  21. struct call_single_data __percpu *csd;
  22. cpumask_var_t cpumask;
  23. cpumask_var_t cpumask_ipi;
  24. };
  25. static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
  26. struct call_single_queue {
  27. struct list_head list;
  28. raw_spinlock_t lock;
  29. };
  30. static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
  31. static int
  32. hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
  33. {
  34. long cpu = (long)hcpu;
  35. struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
  36. switch (action) {
  37. case CPU_UP_PREPARE:
  38. case CPU_UP_PREPARE_FROZEN:
  39. if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
  40. cpu_to_node(cpu)))
  41. return notifier_from_errno(-ENOMEM);
  42. if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
  43. cpu_to_node(cpu))) {
  44. free_cpumask_var(cfd->cpumask);
  45. return notifier_from_errno(-ENOMEM);
  46. }
  47. cfd->csd = alloc_percpu(struct call_single_data);
  48. if (!cfd->csd) {
  49. free_cpumask_var(cfd->cpumask_ipi);
  50. free_cpumask_var(cfd->cpumask);
  51. return notifier_from_errno(-ENOMEM);
  52. }
  53. break;
  54. #ifdef CONFIG_HOTPLUG_CPU
  55. case CPU_UP_CANCELED:
  56. case CPU_UP_CANCELED_FROZEN:
  57. case CPU_DEAD:
  58. case CPU_DEAD_FROZEN:
  59. free_cpumask_var(cfd->cpumask);
  60. free_cpumask_var(cfd->cpumask_ipi);
  61. free_percpu(cfd->csd);
  62. break;
  63. #endif
  64. };
  65. return NOTIFY_OK;
  66. }
  67. static struct notifier_block hotplug_cfd_notifier = {
  68. .notifier_call = hotplug_cfd,
  69. };
  70. void __init call_function_init(void)
  71. {
  72. void *cpu = (void *)(long)smp_processor_id();
  73. int i;
  74. for_each_possible_cpu(i) {
  75. struct call_single_queue *q = &per_cpu(call_single_queue, i);
  76. raw_spin_lock_init(&q->lock);
  77. INIT_LIST_HEAD(&q->list);
  78. }
  79. hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
  80. register_cpu_notifier(&hotplug_cfd_notifier);
  81. }
  82. /*
  83. * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
  84. *
  85. * For non-synchronous ipi calls the csd can still be in use by the
  86. * previous function call. For multi-cpu calls its even more interesting
  87. * as we'll have to ensure no other cpu is observing our csd.
  88. */
  89. static void csd_lock_wait(struct call_single_data *csd)
  90. {
  91. while (csd->flags & CSD_FLAG_LOCK)
  92. cpu_relax();
  93. }
  94. static void csd_lock(struct call_single_data *csd)
  95. {
  96. csd_lock_wait(csd);
  97. csd->flags |= CSD_FLAG_LOCK;
  98. /*
  99. * prevent CPU from reordering the above assignment
  100. * to ->flags with any subsequent assignments to other
  101. * fields of the specified call_single_data structure:
  102. */
  103. smp_mb();
  104. }
  105. static void csd_unlock(struct call_single_data *csd)
  106. {
  107. WARN_ON((csd->flags & CSD_FLAG_WAIT) && !(csd->flags & CSD_FLAG_LOCK));
  108. /*
  109. * ensure we're all done before releasing data:
  110. */
  111. smp_mb();
  112. csd->flags &= ~CSD_FLAG_LOCK;
  113. }
  114. /*
  115. * Insert a previously allocated call_single_data element
  116. * for execution on the given CPU. data must already have
  117. * ->func, ->info, and ->flags set.
  118. */
  119. static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
  120. {
  121. struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
  122. unsigned long flags;
  123. int ipi;
  124. if (wait)
  125. csd->flags |= CSD_FLAG_WAIT;
  126. raw_spin_lock_irqsave(&dst->lock, flags);
  127. ipi = list_empty(&dst->list);
  128. list_add_tail(&csd->list, &dst->list);
  129. raw_spin_unlock_irqrestore(&dst->lock, flags);
  130. /*
  131. * The list addition should be visible before sending the IPI
  132. * handler locks the list to pull the entry off it because of
  133. * normal cache coherency rules implied by spinlocks.
  134. *
  135. * If IPIs can go out of order to the cache coherency protocol
  136. * in an architecture, sufficient synchronisation should be added
  137. * to arch code to make it appear to obey cache coherency WRT
  138. * locking and barrier primitives. Generic code isn't really
  139. * equipped to do the right thing...
  140. */
  141. if (ipi)
  142. arch_send_call_function_single_ipi(cpu);
  143. if (wait)
  144. csd_lock_wait(csd);
  145. }
  146. /*
  147. * Invoked by arch to handle an IPI for call function single. Must be
  148. * called from the arch with interrupts disabled.
  149. */
  150. void generic_smp_call_function_single_interrupt(void)
  151. {
  152. struct call_single_queue *q = &__get_cpu_var(call_single_queue);
  153. LIST_HEAD(list);
  154. /*
  155. * Shouldn't receive this interrupt on a cpu that is not yet online.
  156. */
  157. WARN_ON_ONCE(!cpu_online(smp_processor_id()));
  158. raw_spin_lock(&q->lock);
  159. list_replace_init(&q->list, &list);
  160. raw_spin_unlock(&q->lock);
  161. while (!list_empty(&list)) {
  162. struct call_single_data *csd;
  163. csd = list_entry(list.next, struct call_single_data, list);
  164. list_del(&csd->list);
  165. csd->func(csd->info);
  166. csd_unlock(csd);
  167. }
  168. }
  169. static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
  170. /*
  171. * smp_call_function_single - Run a function on a specific CPU
  172. * @func: The function to run. This must be fast and non-blocking.
  173. * @info: An arbitrary pointer to pass to the function.
  174. * @wait: If true, wait until function has completed on other CPUs.
  175. *
  176. * Returns 0 on success, else a negative status code.
  177. */
  178. int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
  179. int wait)
  180. {
  181. struct call_single_data d = {
  182. .flags = 0,
  183. };
  184. unsigned long flags;
  185. int this_cpu;
  186. int err = 0;
  187. /*
  188. * prevent preemption and reschedule on another processor,
  189. * as well as CPU removal
  190. */
  191. this_cpu = get_cpu();
  192. /*
  193. * Can deadlock when called with interrupts disabled.
  194. * We allow cpu's that are not yet online though, as no one else can
  195. * send smp call function interrupt to this cpu and as such deadlocks
  196. * can't happen.
  197. */
  198. WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
  199. && !oops_in_progress);
  200. if (cpu == this_cpu) {
  201. local_irq_save(flags);
  202. func(info);
  203. local_irq_restore(flags);
  204. } else {
  205. if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
  206. struct call_single_data *csd = &d;
  207. if (!wait)
  208. csd = &__get_cpu_var(csd_data);
  209. csd_lock(csd);
  210. csd->func = func;
  211. csd->info = info;
  212. generic_exec_single(cpu, csd, wait);
  213. } else {
  214. err = -ENXIO; /* CPU not online */
  215. }
  216. }
  217. put_cpu();
  218. return err;
  219. }
  220. EXPORT_SYMBOL(smp_call_function_single);
  221. /*
  222. * smp_call_function_any - Run a function on any of the given cpus
  223. * @mask: The mask of cpus it can run on.
  224. * @func: The function to run. This must be fast and non-blocking.
  225. * @info: An arbitrary pointer to pass to the function.
  226. * @wait: If true, wait until function has completed.
  227. *
  228. * Returns 0 on success, else a negative status code (if no cpus were online).
  229. *
  230. * Selection preference:
  231. * 1) current cpu if in @mask
  232. * 2) any cpu of current node if in @mask
  233. * 3) any other online cpu in @mask
  234. */
  235. int smp_call_function_any(const struct cpumask *mask,
  236. smp_call_func_t func, void *info, int wait)
  237. {
  238. unsigned int cpu;
  239. const struct cpumask *nodemask;
  240. int ret;
  241. /* Try for same CPU (cheapest) */
  242. cpu = get_cpu();
  243. if (cpumask_test_cpu(cpu, mask))
  244. goto call;
  245. /* Try for same node. */
  246. nodemask = cpumask_of_node(cpu_to_node(cpu));
  247. for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
  248. cpu = cpumask_next_and(cpu, nodemask, mask)) {
  249. if (cpu_online(cpu))
  250. goto call;
  251. }
  252. /* Any online will do: smp_call_function_single handles nr_cpu_ids. */
  253. cpu = cpumask_any_and(mask, cpu_online_mask);
  254. call:
  255. ret = smp_call_function_single(cpu, func, info, wait);
  256. put_cpu();
  257. return ret;
  258. }
  259. EXPORT_SYMBOL_GPL(smp_call_function_any);
  260. /**
  261. * __smp_call_function_single(): Run a function on a specific CPU
  262. * @cpu: The CPU to run on.
  263. * @data: Pre-allocated and setup data structure
  264. * @wait: If true, wait until function has completed on specified CPU.
  265. *
  266. * Like smp_call_function_single(), but allow caller to pass in a
  267. * pre-allocated data structure. Useful for embedding @data inside
  268. * other structures, for instance.
  269. */
  270. void __smp_call_function_single(int cpu, struct call_single_data *csd,
  271. int wait)
  272. {
  273. unsigned int this_cpu;
  274. unsigned long flags;
  275. this_cpu = get_cpu();
  276. /*
  277. * Can deadlock when called with interrupts disabled.
  278. * We allow cpu's that are not yet online though, as no one else can
  279. * send smp call function interrupt to this cpu and as such deadlocks
  280. * can't happen.
  281. */
  282. WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
  283. && !oops_in_progress);
  284. if (cpu == this_cpu) {
  285. local_irq_save(flags);
  286. csd->func(csd->info);
  287. local_irq_restore(flags);
  288. } else {
  289. csd_lock(csd);
  290. generic_exec_single(cpu, csd, wait);
  291. }
  292. put_cpu();
  293. }
  294. EXPORT_SYMBOL_GPL(__smp_call_function_single);
  295. /**
  296. * smp_call_function_many(): Run a function on a set of other CPUs.
  297. * @mask: The set of cpus to run on (only runs on online subset).
  298. * @func: The function to run. This must be fast and non-blocking.
  299. * @info: An arbitrary pointer to pass to the function.
  300. * @wait: If true, wait (atomically) until function has completed
  301. * on other CPUs.
  302. *
  303. * If @wait is true, then returns once @func has returned.
  304. *
  305. * You must not call this function with disabled interrupts or from a
  306. * hardware interrupt handler or from a bottom half handler. Preemption
  307. * must be disabled when calling this function.
  308. */
  309. void smp_call_function_many(const struct cpumask *mask,
  310. smp_call_func_t func, void *info, bool wait)
  311. {
  312. struct call_function_data *cfd;
  313. int cpu, next_cpu, this_cpu = smp_processor_id();
  314. /*
  315. * Can deadlock when called with interrupts disabled.
  316. * We allow cpu's that are not yet online though, as no one else can
  317. * send smp call function interrupt to this cpu and as such deadlocks
  318. * can't happen.
  319. */
  320. WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
  321. && !oops_in_progress && !early_boot_irqs_disabled);
  322. /* Try to fastpath. So, what's a CPU they want? Ignoring this one. */
  323. cpu = cpumask_first_and(mask, cpu_online_mask);
  324. if (cpu == this_cpu)
  325. cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
  326. /* No online cpus? We're done. */
  327. if (cpu >= nr_cpu_ids)
  328. return;
  329. /* Do we have another CPU which isn't us? */
  330. next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
  331. if (next_cpu == this_cpu)
  332. next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
  333. /* Fastpath: do that cpu by itself. */
  334. if (next_cpu >= nr_cpu_ids) {
  335. smp_call_function_single(cpu, func, info, wait);
  336. return;
  337. }
  338. cfd = &__get_cpu_var(cfd_data);
  339. cpumask_and(cfd->cpumask, mask, cpu_online_mask);
  340. cpumask_clear_cpu(this_cpu, cfd->cpumask);
  341. /* Some callers race with other cpus changing the passed mask */
  342. if (unlikely(!cpumask_weight(cfd->cpumask)))
  343. return;
  344. /*
  345. * After we put an entry into the list, cfd->cpumask may be cleared
  346. * again when another CPU sends another IPI for a SMP function call, so
  347. * cfd->cpumask will be zero.
  348. */
  349. cpumask_copy(cfd->cpumask_ipi, cfd->cpumask);
  350. for_each_cpu(cpu, cfd->cpumask) {
  351. struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
  352. struct call_single_queue *dst =
  353. &per_cpu(call_single_queue, cpu);
  354. unsigned long flags;
  355. csd_lock(csd);
  356. csd->func = func;
  357. csd->info = info;
  358. raw_spin_lock_irqsave(&dst->lock, flags);
  359. list_add_tail(&csd->list, &dst->list);
  360. raw_spin_unlock_irqrestore(&dst->lock, flags);
  361. }
  362. /* Send a message to all CPUs in the map */
  363. arch_send_call_function_ipi_mask(cfd->cpumask_ipi);
  364. if (wait) {
  365. for_each_cpu(cpu, cfd->cpumask) {
  366. struct call_single_data *csd;
  367. csd = per_cpu_ptr(cfd->csd, cpu);
  368. csd_lock_wait(csd);
  369. }
  370. }
  371. }
  372. EXPORT_SYMBOL(smp_call_function_many);
  373. /**
  374. * smp_call_function(): Run a function on all other CPUs.
  375. * @func: The function to run. This must be fast and non-blocking.
  376. * @info: An arbitrary pointer to pass to the function.
  377. * @wait: If true, wait (atomically) until function has completed
  378. * on other CPUs.
  379. *
  380. * Returns 0.
  381. *
  382. * If @wait is true, then returns once @func has returned; otherwise
  383. * it returns just before the target cpu calls @func.
  384. *
  385. * You must not call this function with disabled interrupts or from a
  386. * hardware interrupt handler or from a bottom half handler.
  387. */
  388. int smp_call_function(smp_call_func_t func, void *info, int wait)
  389. {
  390. preempt_disable();
  391. smp_call_function_many(cpu_online_mask, func, info, wait);
  392. preempt_enable();
  393. return 0;
  394. }
  395. EXPORT_SYMBOL(smp_call_function);
  396. /* Setup configured maximum number of CPUs to activate */
  397. unsigned int setup_max_cpus = NR_CPUS;
  398. EXPORT_SYMBOL(setup_max_cpus);
  399. /*
  400. * Setup routine for controlling SMP activation
  401. *
  402. * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
  403. * activation entirely (the MPS table probe still happens, though).
  404. *
  405. * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
  406. * greater than 0, limits the maximum number of CPUs activated in
  407. * SMP mode to <NUM>.
  408. */
  409. void __weak arch_disable_smp_support(void) { }
  410. static int __init nosmp(char *str)
  411. {
  412. setup_max_cpus = 0;
  413. arch_disable_smp_support();
  414. return 0;
  415. }
  416. early_param("nosmp", nosmp);
  417. /* this is hard limit */
  418. static int __init nrcpus(char *str)
  419. {
  420. int nr_cpus;
  421. get_option(&str, &nr_cpus);
  422. if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
  423. nr_cpu_ids = nr_cpus;
  424. return 0;
  425. }
  426. early_param("nr_cpus", nrcpus);
  427. static int __init maxcpus(char *str)
  428. {
  429. get_option(&str, &setup_max_cpus);
  430. if (setup_max_cpus == 0)
  431. arch_disable_smp_support();
  432. return 0;
  433. }
  434. early_param("maxcpus", maxcpus);
  435. /* Setup number of possible processor ids */
  436. int nr_cpu_ids __read_mostly = NR_CPUS;
  437. EXPORT_SYMBOL(nr_cpu_ids);
  438. /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
  439. void __init setup_nr_cpu_ids(void)
  440. {
  441. nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
  442. }
  443. void __weak smp_announce(void)
  444. {
  445. printk(KERN_INFO "Brought up %d CPUs\n", num_online_cpus());
  446. }
  447. /* Called by boot processor to activate the rest. */
  448. void __init smp_init(void)
  449. {
  450. unsigned int cpu;
  451. idle_threads_init();
  452. /* FIXME: This should be done in userspace --RR */
  453. for_each_present_cpu(cpu) {
  454. if (num_online_cpus() >= setup_max_cpus)
  455. break;
  456. if (!cpu_online(cpu))
  457. cpu_up(cpu);
  458. }
  459. /* Any cleanup work */
  460. smp_announce();
  461. smp_cpus_done(setup_max_cpus);
  462. }
  463. /*
  464. * Call a function on all processors. May be used during early boot while
  465. * early_boot_irqs_disabled is set. Use local_irq_save/restore() instead
  466. * of local_irq_disable/enable().
  467. */
  468. int on_each_cpu(void (*func) (void *info), void *info, int wait)
  469. {
  470. unsigned long flags;
  471. int ret = 0;
  472. preempt_disable();
  473. ret = smp_call_function(func, info, wait);
  474. local_irq_save(flags);
  475. func(info);
  476. local_irq_restore(flags);
  477. preempt_enable();
  478. return ret;
  479. }
  480. EXPORT_SYMBOL(on_each_cpu);
  481. /**
  482. * on_each_cpu_mask(): Run a function on processors specified by
  483. * cpumask, which may include the local processor.
  484. * @mask: The set of cpus to run on (only runs on online subset).
  485. * @func: The function to run. This must be fast and non-blocking.
  486. * @info: An arbitrary pointer to pass to the function.
  487. * @wait: If true, wait (atomically) until function has completed
  488. * on other CPUs.
  489. *
  490. * If @wait is true, then returns once @func has returned.
  491. *
  492. * You must not call this function with disabled interrupts or from a
  493. * hardware interrupt handler or from a bottom half handler. The
  494. * exception is that it may be used during early boot while
  495. * early_boot_irqs_disabled is set.
  496. */
  497. void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func,
  498. void *info, bool wait)
  499. {
  500. int cpu = get_cpu();
  501. smp_call_function_many(mask, func, info, wait);
  502. if (cpumask_test_cpu(cpu, mask)) {
  503. unsigned long flags;
  504. local_irq_save(flags);
  505. func(info);
  506. local_irq_restore(flags);
  507. }
  508. put_cpu();
  509. }
  510. EXPORT_SYMBOL(on_each_cpu_mask);
  511. /*
  512. * on_each_cpu_cond(): Call a function on each processor for which
  513. * the supplied function cond_func returns true, optionally waiting
  514. * for all the required CPUs to finish. This may include the local
  515. * processor.
  516. * @cond_func: A callback function that is passed a cpu id and
  517. * the the info parameter. The function is called
  518. * with preemption disabled. The function should
  519. * return a blooean value indicating whether to IPI
  520. * the specified CPU.
  521. * @func: The function to run on all applicable CPUs.
  522. * This must be fast and non-blocking.
  523. * @info: An arbitrary pointer to pass to both functions.
  524. * @wait: If true, wait (atomically) until function has
  525. * completed on other CPUs.
  526. * @gfp_flags: GFP flags to use when allocating the cpumask
  527. * used internally by the function.
  528. *
  529. * The function might sleep if the GFP flags indicates a non
  530. * atomic allocation is allowed.
  531. *
  532. * Preemption is disabled to protect against CPUs going offline but not online.
  533. * CPUs going online during the call will not be seen or sent an IPI.
  534. *
  535. * You must not call this function with disabled interrupts or
  536. * from a hardware interrupt handler or from a bottom half handler.
  537. */
  538. void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
  539. smp_call_func_t func, void *info, bool wait,
  540. gfp_t gfp_flags)
  541. {
  542. cpumask_var_t cpus;
  543. int cpu, ret;
  544. might_sleep_if(gfp_flags & __GFP_WAIT);
  545. if (likely(zalloc_cpumask_var(&cpus, (gfp_flags|__GFP_NOWARN)))) {
  546. preempt_disable();
  547. for_each_online_cpu(cpu)
  548. if (cond_func(cpu, info))
  549. cpumask_set_cpu(cpu, cpus);
  550. on_each_cpu_mask(cpus, func, info, wait);
  551. preempt_enable();
  552. free_cpumask_var(cpus);
  553. } else {
  554. /*
  555. * No free cpumask, bother. No matter, we'll
  556. * just have to IPI them one by one.
  557. */
  558. preempt_disable();
  559. for_each_online_cpu(cpu)
  560. if (cond_func(cpu, info)) {
  561. ret = smp_call_function_single(cpu, func,
  562. info, wait);
  563. WARN_ON_ONCE(!ret);
  564. }
  565. preempt_enable();
  566. }
  567. }
  568. EXPORT_SYMBOL(on_each_cpu_cond);
  569. static void do_nothing(void *unused)
  570. {
  571. }
  572. /**
  573. * kick_all_cpus_sync - Force all cpus out of idle
  574. *
  575. * Used to synchronize the update of pm_idle function pointer. It's
  576. * called after the pointer is updated and returns after the dummy
  577. * callback function has been executed on all cpus. The execution of
  578. * the function can only happen on the remote cpus after they have
  579. * left the idle function which had been called via pm_idle function
  580. * pointer. So it's guaranteed that nothing uses the previous pointer
  581. * anymore.
  582. */
  583. void kick_all_cpus_sync(void)
  584. {
  585. /* Make sure the change is visible before we kick the cpus */
  586. smp_mb();
  587. smp_call_function(do_nothing, NULL, 1);
  588. }
  589. EXPORT_SYMBOL_GPL(kick_all_cpus_sync);