cpu.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597
  1. /* CPU control.
  2. * (C) 2001, 2002, 2003, 2004 Rusty Russell
  3. *
  4. * This code is licenced under the GPL.
  5. */
  6. #include <linux/proc_fs.h>
  7. #include <linux/smp.h>
  8. #include <linux/init.h>
  9. #include <linux/notifier.h>
  10. #include <linux/sched.h>
  11. #include <linux/unistd.h>
  12. #include <linux/cpu.h>
  13. #include <linux/module.h>
  14. #include <linux/kthread.h>
  15. #include <linux/stop_machine.h>
  16. #include <linux/mutex.h>
  17. #include <linux/gfp.h>
  18. #ifdef CONFIG_SMP
  19. /* Serializes the updates to cpu_online_mask, cpu_present_mask */
  20. static DEFINE_MUTEX(cpu_add_remove_lock);
  21. /*
  22. * The following two API's must be used when attempting
  23. * to serialize the updates to cpu_online_mask, cpu_present_mask.
  24. */
  25. void cpu_maps_update_begin(void)
  26. {
  27. mutex_lock(&cpu_add_remove_lock);
  28. }
  29. void cpu_maps_update_done(void)
  30. {
  31. mutex_unlock(&cpu_add_remove_lock);
  32. }
  33. static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
  34. /* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
  35. * Should always be manipulated under cpu_add_remove_lock
  36. */
  37. static int cpu_hotplug_disabled;
  38. #ifdef CONFIG_HOTPLUG_CPU
  39. static struct {
  40. struct task_struct *active_writer;
  41. struct mutex lock; /* Synchronizes accesses to refcount, */
  42. /*
  43. * Also blocks the new readers during
  44. * an ongoing cpu hotplug operation.
  45. */
  46. int refcount;
  47. } cpu_hotplug = {
  48. .active_writer = NULL,
  49. .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
  50. .refcount = 0,
  51. };
  52. void get_online_cpus(void)
  53. {
  54. might_sleep();
  55. if (cpu_hotplug.active_writer == current)
  56. return;
  57. mutex_lock(&cpu_hotplug.lock);
  58. cpu_hotplug.refcount++;
  59. mutex_unlock(&cpu_hotplug.lock);
  60. }
  61. EXPORT_SYMBOL_GPL(get_online_cpus);
  62. void put_online_cpus(void)
  63. {
  64. if (cpu_hotplug.active_writer == current)
  65. return;
  66. mutex_lock(&cpu_hotplug.lock);
  67. if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
  68. wake_up_process(cpu_hotplug.active_writer);
  69. mutex_unlock(&cpu_hotplug.lock);
  70. }
  71. EXPORT_SYMBOL_GPL(put_online_cpus);
  72. /*
  73. * This ensures that the hotplug operation can begin only when the
  74. * refcount goes to zero.
  75. *
  76. * Note that during a cpu-hotplug operation, the new readers, if any,
  77. * will be blocked by the cpu_hotplug.lock
  78. *
  79. * Since cpu_hotplug_begin() is always called after invoking
  80. * cpu_maps_update_begin(), we can be sure that only one writer is active.
  81. *
  82. * Note that theoretically, there is a possibility of a livelock:
  83. * - Refcount goes to zero, last reader wakes up the sleeping
  84. * writer.
  85. * - Last reader unlocks the cpu_hotplug.lock.
  86. * - A new reader arrives at this moment, bumps up the refcount.
  87. * - The writer acquires the cpu_hotplug.lock finds the refcount
  88. * non zero and goes to sleep again.
  89. *
  90. * However, this is very difficult to achieve in practice since
  91. * get_online_cpus() not an api which is called all that often.
  92. *
  93. */
  94. static void cpu_hotplug_begin(void)
  95. {
  96. cpu_hotplug.active_writer = current;
  97. for (;;) {
  98. mutex_lock(&cpu_hotplug.lock);
  99. if (likely(!cpu_hotplug.refcount))
  100. break;
  101. __set_current_state(TASK_UNINTERRUPTIBLE);
  102. mutex_unlock(&cpu_hotplug.lock);
  103. schedule();
  104. }
  105. }
  106. static void cpu_hotplug_done(void)
  107. {
  108. cpu_hotplug.active_writer = NULL;
  109. mutex_unlock(&cpu_hotplug.lock);
  110. }
  111. #else /* #if CONFIG_HOTPLUG_CPU */
  112. static void cpu_hotplug_begin(void) {}
  113. static void cpu_hotplug_done(void) {}
  114. #endif /* #esle #if CONFIG_HOTPLUG_CPU */
  115. /* Need to know about CPUs going up/down? */
  116. int __ref register_cpu_notifier(struct notifier_block *nb)
  117. {
  118. int ret;
  119. cpu_maps_update_begin();
  120. ret = raw_notifier_chain_register(&cpu_chain, nb);
  121. cpu_maps_update_done();
  122. return ret;
  123. }
  124. static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
  125. int *nr_calls)
  126. {
  127. int ret;
  128. ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
  129. nr_calls);
  130. return notifier_to_errno(ret);
  131. }
  132. static int cpu_notify(unsigned long val, void *v)
  133. {
  134. return __cpu_notify(val, v, -1, NULL);
  135. }
  136. static void cpu_notify_nofail(unsigned long val, void *v)
  137. {
  138. int err;
  139. err = cpu_notify(val, v);
  140. BUG_ON(err);
  141. }
  142. #ifdef CONFIG_HOTPLUG_CPU
  143. EXPORT_SYMBOL(register_cpu_notifier);
  144. void __ref unregister_cpu_notifier(struct notifier_block *nb)
  145. {
  146. cpu_maps_update_begin();
  147. raw_notifier_chain_unregister(&cpu_chain, nb);
  148. cpu_maps_update_done();
  149. }
  150. EXPORT_SYMBOL(unregister_cpu_notifier);
  151. static inline void check_for_tasks(int cpu)
  152. {
  153. struct task_struct *p;
  154. write_lock_irq(&tasklist_lock);
  155. for_each_process(p) {
  156. if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
  157. (!cputime_eq(p->utime, cputime_zero) ||
  158. !cputime_eq(p->stime, cputime_zero)))
  159. printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
  160. "(state = %ld, flags = %x)\n",
  161. p->comm, task_pid_nr(p), cpu,
  162. p->state, p->flags);
  163. }
  164. write_unlock_irq(&tasklist_lock);
  165. }
  166. struct take_cpu_down_param {
  167. struct task_struct *caller;
  168. unsigned long mod;
  169. void *hcpu;
  170. };
  171. /* Take this CPU down. */
  172. static int __ref take_cpu_down(void *_param)
  173. {
  174. struct take_cpu_down_param *param = _param;
  175. unsigned int cpu = (unsigned long)param->hcpu;
  176. int err;
  177. /* Ensure this CPU doesn't handle any more interrupts. */
  178. err = __cpu_disable();
  179. if (err < 0)
  180. return err;
  181. cpu_notify(CPU_DYING | param->mod, param->hcpu);
  182. if (task_cpu(param->caller) == cpu)
  183. move_task_off_dead_cpu(cpu, param->caller);
  184. /* Force idle task to run as soon as we yield: it should
  185. immediately notice cpu is offline and die quickly. */
  186. sched_idle_next();
  187. return 0;
  188. }
  189. /* Requires cpu_add_remove_lock to be held */
  190. static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
  191. {
  192. int err, nr_calls = 0;
  193. void *hcpu = (void *)(long)cpu;
  194. unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
  195. struct take_cpu_down_param tcd_param = {
  196. .caller = current,
  197. .mod = mod,
  198. .hcpu = hcpu,
  199. };
  200. if (num_online_cpus() == 1)
  201. return -EBUSY;
  202. if (!cpu_online(cpu))
  203. return -EINVAL;
  204. cpu_hotplug_begin();
  205. set_cpu_active(cpu, false);
  206. err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
  207. if (err) {
  208. set_cpu_active(cpu, true);
  209. nr_calls--;
  210. __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
  211. printk("%s: attempt to take down CPU %u failed\n",
  212. __func__, cpu);
  213. goto out_release;
  214. }
  215. err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
  216. if (err) {
  217. set_cpu_active(cpu, true);
  218. /* CPU didn't die: tell everyone. Can't complain. */
  219. cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
  220. goto out_release;
  221. }
  222. BUG_ON(cpu_online(cpu));
  223. /* Wait for it to sleep (leaving idle task). */
  224. while (!idle_cpu(cpu))
  225. yield();
  226. /* This actually kills the CPU. */
  227. __cpu_die(cpu);
  228. /* CPU is completely dead: tell everyone. Too late to complain. */
  229. cpu_notify_nofail(CPU_DEAD | mod, hcpu);
  230. check_for_tasks(cpu);
  231. out_release:
  232. cpu_hotplug_done();
  233. if (!err)
  234. cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
  235. return err;
  236. }
  237. int __ref cpu_down(unsigned int cpu)
  238. {
  239. int err;
  240. cpu_maps_update_begin();
  241. if (cpu_hotplug_disabled) {
  242. err = -EBUSY;
  243. goto out;
  244. }
  245. err = _cpu_down(cpu, 0);
  246. out:
  247. cpu_maps_update_done();
  248. return err;
  249. }
  250. EXPORT_SYMBOL(cpu_down);
  251. #endif /*CONFIG_HOTPLUG_CPU*/
  252. /* Requires cpu_add_remove_lock to be held */
  253. static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
  254. {
  255. int ret, nr_calls = 0;
  256. void *hcpu = (void *)(long)cpu;
  257. unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
  258. if (cpu_online(cpu) || !cpu_present(cpu))
  259. return -EINVAL;
  260. cpu_hotplug_begin();
  261. ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
  262. if (ret) {
  263. nr_calls--;
  264. printk("%s: attempt to bring up CPU %u failed\n",
  265. __func__, cpu);
  266. goto out_notify;
  267. }
  268. /* Arch-specific enabling code. */
  269. ret = __cpu_up(cpu);
  270. if (ret != 0)
  271. goto out_notify;
  272. BUG_ON(!cpu_online(cpu));
  273. set_cpu_active(cpu, true);
  274. /* Now call notifier in preparation. */
  275. cpu_notify(CPU_ONLINE | mod, hcpu);
  276. out_notify:
  277. if (ret != 0)
  278. __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
  279. cpu_hotplug_done();
  280. return ret;
  281. }
  282. int __cpuinit cpu_up(unsigned int cpu)
  283. {
  284. int err = 0;
  285. #ifdef CONFIG_MEMORY_HOTPLUG
  286. int nid;
  287. pg_data_t *pgdat;
  288. #endif
  289. if (!cpu_possible(cpu)) {
  290. printk(KERN_ERR "can't online cpu %d because it is not "
  291. "configured as may-hotadd at boot time\n", cpu);
  292. #if defined(CONFIG_IA64)
  293. printk(KERN_ERR "please check additional_cpus= boot "
  294. "parameter\n");
  295. #endif
  296. return -EINVAL;
  297. }
  298. #ifdef CONFIG_MEMORY_HOTPLUG
  299. nid = cpu_to_node(cpu);
  300. if (!node_online(nid)) {
  301. err = mem_online_node(nid);
  302. if (err)
  303. return err;
  304. }
  305. pgdat = NODE_DATA(nid);
  306. if (!pgdat) {
  307. printk(KERN_ERR
  308. "Can't online cpu %d due to NULL pgdat\n", cpu);
  309. return -ENOMEM;
  310. }
  311. if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
  312. mutex_lock(&zonelists_mutex);
  313. build_all_zonelists(NULL);
  314. mutex_unlock(&zonelists_mutex);
  315. }
  316. #endif
  317. cpu_maps_update_begin();
  318. if (cpu_hotplug_disabled) {
  319. err = -EBUSY;
  320. goto out;
  321. }
  322. err = _cpu_up(cpu, 0);
  323. out:
  324. cpu_maps_update_done();
  325. return err;
  326. }
  327. #ifdef CONFIG_PM_SLEEP_SMP
  328. static cpumask_var_t frozen_cpus;
  329. int disable_nonboot_cpus(void)
  330. {
  331. int cpu, first_cpu, error;
  332. cpu_maps_update_begin();
  333. first_cpu = cpumask_first(cpu_online_mask);
  334. /*
  335. * We take down all of the non-boot CPUs in one shot to avoid races
  336. * with the userspace trying to use the CPU hotplug at the same time
  337. */
  338. cpumask_clear(frozen_cpus);
  339. printk("Disabling non-boot CPUs ...\n");
  340. for_each_online_cpu(cpu) {
  341. if (cpu == first_cpu)
  342. continue;
  343. error = _cpu_down(cpu, 1);
  344. if (!error)
  345. cpumask_set_cpu(cpu, frozen_cpus);
  346. else {
  347. printk(KERN_ERR "Error taking CPU%d down: %d\n",
  348. cpu, error);
  349. break;
  350. }
  351. }
  352. if (!error) {
  353. BUG_ON(num_online_cpus() > 1);
  354. /* Make sure the CPUs won't be enabled by someone else */
  355. cpu_hotplug_disabled = 1;
  356. } else {
  357. printk(KERN_ERR "Non-boot CPUs are not disabled\n");
  358. }
  359. cpu_maps_update_done();
  360. return error;
  361. }
  362. void __weak arch_enable_nonboot_cpus_begin(void)
  363. {
  364. }
  365. void __weak arch_enable_nonboot_cpus_end(void)
  366. {
  367. }
  368. void __ref enable_nonboot_cpus(void)
  369. {
  370. int cpu, error;
  371. /* Allow everyone to use the CPU hotplug again */
  372. cpu_maps_update_begin();
  373. cpu_hotplug_disabled = 0;
  374. if (cpumask_empty(frozen_cpus))
  375. goto out;
  376. printk("Enabling non-boot CPUs ...\n");
  377. arch_enable_nonboot_cpus_begin();
  378. for_each_cpu(cpu, frozen_cpus) {
  379. error = _cpu_up(cpu, 1);
  380. if (!error) {
  381. printk("CPU%d is up\n", cpu);
  382. continue;
  383. }
  384. printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
  385. }
  386. arch_enable_nonboot_cpus_end();
  387. cpumask_clear(frozen_cpus);
  388. out:
  389. cpu_maps_update_done();
  390. }
  391. static int alloc_frozen_cpus(void)
  392. {
  393. if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
  394. return -ENOMEM;
  395. return 0;
  396. }
  397. core_initcall(alloc_frozen_cpus);
  398. #endif /* CONFIG_PM_SLEEP_SMP */
  399. /**
  400. * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
  401. * @cpu: cpu that just started
  402. *
  403. * This function calls the cpu_chain notifiers with CPU_STARTING.
  404. * It must be called by the arch code on the new cpu, before the new cpu
  405. * enables interrupts and before the "boot" cpu returns from __cpu_up().
  406. */
  407. void __cpuinit notify_cpu_starting(unsigned int cpu)
  408. {
  409. unsigned long val = CPU_STARTING;
  410. #ifdef CONFIG_PM_SLEEP_SMP
  411. if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
  412. val = CPU_STARTING_FROZEN;
  413. #endif /* CONFIG_PM_SLEEP_SMP */
  414. cpu_notify(val, (void *)(long)cpu);
  415. }
  416. #endif /* CONFIG_SMP */
  417. /*
  418. * cpu_bit_bitmap[] is a special, "compressed" data structure that
  419. * represents all NR_CPUS bits binary values of 1<<nr.
  420. *
  421. * It is used by cpumask_of() to get a constant address to a CPU
  422. * mask value that has a single bit set only.
  423. */
  424. /* cpu_bit_bitmap[0] is empty - so we can back into it */
  425. #define MASK_DECLARE_1(x) [x+1][0] = 1UL << (x)
  426. #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
  427. #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
  428. #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
  429. const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
  430. MASK_DECLARE_8(0), MASK_DECLARE_8(8),
  431. MASK_DECLARE_8(16), MASK_DECLARE_8(24),
  432. #if BITS_PER_LONG > 32
  433. MASK_DECLARE_8(32), MASK_DECLARE_8(40),
  434. MASK_DECLARE_8(48), MASK_DECLARE_8(56),
  435. #endif
  436. };
  437. EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
  438. const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
  439. EXPORT_SYMBOL(cpu_all_bits);
  440. #ifdef CONFIG_INIT_ALL_POSSIBLE
  441. static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
  442. = CPU_BITS_ALL;
  443. #else
  444. static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
  445. #endif
  446. const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
  447. EXPORT_SYMBOL(cpu_possible_mask);
  448. static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
  449. const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
  450. EXPORT_SYMBOL(cpu_online_mask);
  451. static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
  452. const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
  453. EXPORT_SYMBOL(cpu_present_mask);
  454. static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
  455. const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
  456. EXPORT_SYMBOL(cpu_active_mask);
  457. void set_cpu_possible(unsigned int cpu, bool possible)
  458. {
  459. if (possible)
  460. cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
  461. else
  462. cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
  463. }
  464. void set_cpu_present(unsigned int cpu, bool present)
  465. {
  466. if (present)
  467. cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
  468. else
  469. cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
  470. }
  471. void set_cpu_online(unsigned int cpu, bool online)
  472. {
  473. if (online)
  474. cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
  475. else
  476. cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
  477. }
  478. void set_cpu_active(unsigned int cpu, bool active)
  479. {
  480. if (active)
  481. cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
  482. else
  483. cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
  484. }
  485. void init_cpu_present(const struct cpumask *src)
  486. {
  487. cpumask_copy(to_cpumask(cpu_present_bits), src);
  488. }
  489. void init_cpu_possible(const struct cpumask *src)
  490. {
  491. cpumask_copy(to_cpumask(cpu_possible_bits), src);
  492. }
  493. void init_cpu_online(const struct cpumask *src)
  494. {
  495. cpumask_copy(to_cpumask(cpu_online_bits), src);
  496. }