smpboot_64.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142
  1. /*
  2. * x86 SMP booting functions
  3. *
  4. * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
  5. * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
  6. * Copyright 2001 Andi Kleen, SuSE Labs.
  7. *
  8. * Much of the core SMP work is based on previous work by Thomas Radke, to
  9. * whom a great many thanks are extended.
  10. *
  11. * Thanks to Intel for making available several different Pentium,
  12. * Pentium Pro and Pentium-II/Xeon MP machines.
  13. * Original development of Linux SMP code supported by Caldera.
  14. *
  15. * This code is released under the GNU General Public License version 2
  16. *
  17. * Fixes
  18. * Felix Koop : NR_CPUS used properly
  19. * Jose Renau : Handle single CPU case.
  20. * Alan Cox : By repeated request 8) - Total BogoMIP report.
  21. * Greg Wright : Fix for kernel stacks panic.
  22. * Erich Boleyn : MP v1.4 and additional changes.
  23. * Matthias Sattler : Changes for 2.1 kernel map.
  24. * Michel Lespinasse : Changes for 2.1 kernel map.
  25. * Michael Chastain : Change trampoline.S to gnu as.
  26. * Alan Cox : Dumb bug: 'B' step PPro's are fine
  27. * Ingo Molnar : Added APIC timers, based on code
  28. * from Jose Renau
  29. * Ingo Molnar : various cleanups and rewrites
  30. * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
  31. * Maciej W. Rozycki : Bits for genuine 82489DX APICs
  32. * Andi Kleen : Changed for SMP boot into long mode.
  33. * Rusty Russell : Hacked into shape for new "hotplug" boot process.
  34. * Andi Kleen : Converted to new state machine.
  35. * Various cleanups.
  36. * Probably mostly hotplug CPU ready now.
  37. * Ashok Raj : CPU hotplug support
  38. */
  39. #include <linux/init.h>
  40. #include <linux/mm.h>
  41. #include <linux/kernel_stat.h>
  42. #include <linux/bootmem.h>
  43. #include <linux/thread_info.h>
  44. #include <linux/module.h>
  45. #include <linux/delay.h>
  46. #include <linux/mc146818rtc.h>
  47. #include <linux/smp.h>
  48. #include <linux/kdebug.h>
  49. #include <asm/mtrr.h>
  50. #include <asm/pgalloc.h>
  51. #include <asm/desc.h>
  52. #include <asm/tlbflush.h>
  53. #include <asm/proto.h>
  54. #include <asm/nmi.h>
  55. #include <asm/irq.h>
  56. #include <asm/hw_irq.h>
  57. #include <asm/numa.h>
  58. /* Number of siblings per CPU package */
  59. int smp_num_siblings = 1;
  60. EXPORT_SYMBOL(smp_num_siblings);
  61. /* Last level cache ID of each logical CPU */
  62. DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
  63. /* Bitmask of currently online CPUs */
  64. cpumask_t cpu_online_map __read_mostly;
  65. EXPORT_SYMBOL(cpu_online_map);
  66. /*
  67. * Private maps to synchronize booting between AP and BP.
  68. * Probably not needed anymore, but it makes for easier debugging. -AK
  69. */
  70. cpumask_t cpu_callin_map;
  71. cpumask_t cpu_callout_map;
  72. cpumask_t cpu_possible_map;
  73. EXPORT_SYMBOL(cpu_possible_map);
  74. /* Per CPU bogomips and other parameters */
  75. DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
  76. EXPORT_PER_CPU_SYMBOL(cpu_info);
  77. /* Set when the idlers are all forked */
  78. int smp_threads_ready;
  79. /* representing HT siblings of each logical CPU */
  80. DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
  81. EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  82. /* representing HT and core siblings of each logical CPU */
  83. DEFINE_PER_CPU(cpumask_t, cpu_core_map);
  84. EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  85. /*
  86. * Trampoline 80x86 program as an array.
  87. */
  88. extern const unsigned char trampoline_data[];
  89. extern const unsigned char trampoline_end[];
  90. /* State of each CPU */
  91. DEFINE_PER_CPU(int, cpu_state) = { 0 };
  92. /*
  93. * Store all idle threads, this can be reused instead of creating
  94. * a new thread. Also avoids complicated thread destroy functionality
  95. * for idle threads.
  96. */
  97. #ifdef CONFIG_HOTPLUG_CPU
  98. /*
  99. * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
  100. * removed after init for !CONFIG_HOTPLUG_CPU.
  101. */
  102. static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
  103. #define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
  104. #define set_idle_for_cpu(x,p) (per_cpu(idle_thread_array, x) = (p))
  105. #else
  106. struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
  107. #define get_idle_for_cpu(x) (idle_thread_array[(x)])
  108. #define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
  109. #endif
  110. /*
  111. * Currently trivial. Write the real->protected mode
  112. * bootstrap into the page concerned. The caller
  113. * has made sure it's suitably aligned.
  114. */
  115. static unsigned long __cpuinit setup_trampoline(void)
  116. {
  117. void *tramp = __va(SMP_TRAMPOLINE_BASE);
  118. memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
  119. return virt_to_phys(tramp);
  120. }
  121. /*
  122. * The bootstrap kernel entry code has set these up. Save them for
  123. * a given CPU
  124. */
  125. static void __cpuinit smp_store_cpu_info(int id)
  126. {
  127. struct cpuinfo_x86 *c = &cpu_data(id);
  128. *c = boot_cpu_data;
  129. c->cpu_index = id;
  130. identify_cpu(c);
  131. print_cpu_info(c);
  132. }
  133. static atomic_t init_deasserted __cpuinitdata;
  134. /*
  135. * Report back to the Boot Processor.
  136. * Running on AP.
  137. */
  138. void __cpuinit smp_callin(void)
  139. {
  140. int cpuid, phys_id;
  141. unsigned long timeout;
  142. /*
  143. * If waken up by an INIT in an 82489DX configuration
  144. * we may get here before an INIT-deassert IPI reaches
  145. * our local APIC. We have to wait for the IPI or we'll
  146. * lock up on an APIC access.
  147. */
  148. while (!atomic_read(&init_deasserted))
  149. cpu_relax();
  150. /*
  151. * (This works even if the APIC is not enabled.)
  152. */
  153. phys_id = GET_APIC_ID(apic_read(APIC_ID));
  154. cpuid = smp_processor_id();
  155. if (cpu_isset(cpuid, cpu_callin_map)) {
  156. panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
  157. phys_id, cpuid);
  158. }
  159. Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
  160. /*
  161. * STARTUP IPIs are fragile beasts as they might sometimes
  162. * trigger some glue motherboard logic. Complete APIC bus
  163. * silence for 1 second, this overestimates the time the
  164. * boot CPU is spending to send the up to 2 STARTUP IPIs
  165. * by a factor of two. This should be enough.
  166. */
  167. /*
  168. * Waiting 2s total for startup (udelay is not yet working)
  169. */
  170. timeout = jiffies + 2*HZ;
  171. while (time_before(jiffies, timeout)) {
  172. /*
  173. * Has the boot CPU finished it's STARTUP sequence?
  174. */
  175. if (cpu_isset(cpuid, cpu_callout_map))
  176. break;
  177. cpu_relax();
  178. }
  179. if (!time_before(jiffies, timeout)) {
  180. panic("smp_callin: CPU%d started up but did not get a callout!\n",
  181. cpuid);
  182. }
  183. /*
  184. * the boot CPU has finished the init stage and is spinning
  185. * on callin_map until we finish. We are free to set up this
  186. * CPU, first the APIC. (this is probably redundant on most
  187. * boards)
  188. */
  189. Dprintk("CALLIN, before setup_local_APIC().\n");
  190. setup_local_APIC();
  191. end_local_APIC_setup();
  192. /*
  193. * Get our bogomips.
  194. *
  195. * Need to enable IRQs because it can take longer and then
  196. * the NMI watchdog might kill us.
  197. */
  198. local_irq_enable();
  199. calibrate_delay();
  200. local_irq_disable();
  201. Dprintk("Stack at about %p\n",&cpuid);
  202. /*
  203. * Save our processor parameters
  204. */
  205. smp_store_cpu_info(cpuid);
  206. /*
  207. * Allow the master to continue.
  208. */
  209. cpu_set(cpuid, cpu_callin_map);
  210. }
  211. /* maps the cpu to the sched domain representing multi-core */
  212. cpumask_t cpu_coregroup_map(int cpu)
  213. {
  214. struct cpuinfo_x86 *c = &cpu_data(cpu);
  215. /*
  216. * For perf, we return last level cache shared map.
  217. * And for power savings, we return cpu_core_map
  218. */
  219. if (sched_mc_power_savings || sched_smt_power_savings)
  220. return per_cpu(cpu_core_map, cpu);
  221. else
  222. return c->llc_shared_map;
  223. }
  224. /* representing cpus for which sibling maps can be computed */
  225. static cpumask_t cpu_sibling_setup_map;
  226. static inline void set_cpu_sibling_map(int cpu)
  227. {
  228. int i;
  229. struct cpuinfo_x86 *c = &cpu_data(cpu);
  230. cpu_set(cpu, cpu_sibling_setup_map);
  231. if (smp_num_siblings > 1) {
  232. for_each_cpu_mask(i, cpu_sibling_setup_map) {
  233. if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
  234. c->cpu_core_id == cpu_data(i).cpu_core_id) {
  235. cpu_set(i, per_cpu(cpu_sibling_map, cpu));
  236. cpu_set(cpu, per_cpu(cpu_sibling_map, i));
  237. cpu_set(i, per_cpu(cpu_core_map, cpu));
  238. cpu_set(cpu, per_cpu(cpu_core_map, i));
  239. cpu_set(i, c->llc_shared_map);
  240. cpu_set(cpu, cpu_data(i).llc_shared_map);
  241. }
  242. }
  243. } else {
  244. cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
  245. }
  246. cpu_set(cpu, c->llc_shared_map);
  247. if (current_cpu_data.x86_max_cores == 1) {
  248. per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
  249. c->booted_cores = 1;
  250. return;
  251. }
  252. for_each_cpu_mask(i, cpu_sibling_setup_map) {
  253. if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
  254. per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
  255. cpu_set(i, c->llc_shared_map);
  256. cpu_set(cpu, cpu_data(i).llc_shared_map);
  257. }
  258. if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
  259. cpu_set(i, per_cpu(cpu_core_map, cpu));
  260. cpu_set(cpu, per_cpu(cpu_core_map, i));
  261. /*
  262. * Does this new cpu bringup a new core?
  263. */
  264. if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
  265. /*
  266. * for each core in package, increment
  267. * the booted_cores for this new cpu
  268. */
  269. if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
  270. c->booted_cores++;
  271. /*
  272. * increment the core count for all
  273. * the other cpus in this package
  274. */
  275. if (i != cpu)
  276. cpu_data(i).booted_cores++;
  277. } else if (i != cpu && !c->booted_cores)
  278. c->booted_cores = cpu_data(i).booted_cores;
  279. }
  280. }
  281. }
  282. /*
  283. * Setup code on secondary processor (after comming out of the trampoline)
  284. */
  285. void __cpuinit start_secondary(void)
  286. {
  287. /*
  288. * Dont put anything before smp_callin(), SMP
  289. * booting is too fragile that we want to limit the
  290. * things done here to the most necessary things.
  291. */
  292. cpu_init();
  293. preempt_disable();
  294. smp_callin();
  295. /* otherwise gcc will move up the smp_processor_id before the cpu_init */
  296. barrier();
  297. /*
  298. * Check TSC sync first:
  299. */
  300. check_tsc_sync_target();
  301. if (nmi_watchdog == NMI_IO_APIC) {
  302. disable_8259A_irq(0);
  303. enable_NMI_through_LVT0();
  304. enable_8259A_irq(0);
  305. }
  306. /*
  307. * The sibling maps must be set before turing the online map on for
  308. * this cpu
  309. */
  310. set_cpu_sibling_map(smp_processor_id());
  311. /*
  312. * We need to hold call_lock, so there is no inconsistency
  313. * between the time smp_call_function() determines number of
  314. * IPI recipients, and the time when the determination is made
  315. * for which cpus receive the IPI in genapic_flat.c. Holding this
  316. * lock helps us to not include this cpu in a currently in progress
  317. * smp_call_function().
  318. */
  319. lock_ipi_call_lock();
  320. spin_lock(&vector_lock);
  321. /* Setup the per cpu irq handling data structures */
  322. __setup_vector_irq(smp_processor_id());
  323. /*
  324. * Allow the master to continue.
  325. */
  326. cpu_set(smp_processor_id(), cpu_online_map);
  327. per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
  328. spin_unlock(&vector_lock);
  329. unlock_ipi_call_lock();
  330. setup_secondary_clock();
  331. cpu_idle();
  332. }
  333. extern volatile unsigned long init_rsp;
  334. extern void (*initial_code)(void);
  335. #ifdef APIC_DEBUG
  336. static void inquire_remote_apic(int apicid)
  337. {
  338. unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
  339. char *names[] = { "ID", "VERSION", "SPIV" };
  340. int timeout;
  341. u32 status;
  342. printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
  343. for (i = 0; i < ARRAY_SIZE(regs); i++) {
  344. printk(KERN_INFO "... APIC #%d %s: ", apicid, names[i]);
  345. /*
  346. * Wait for idle.
  347. */
  348. status = safe_apic_wait_icr_idle();
  349. if (status)
  350. printk(KERN_CONT
  351. "a previous APIC delivery may have failed\n");
  352. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
  353. apic_write(APIC_ICR, APIC_DM_REMRD | regs[i]);
  354. timeout = 0;
  355. do {
  356. udelay(100);
  357. status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
  358. } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
  359. switch (status) {
  360. case APIC_ICR_RR_VALID:
  361. status = apic_read(APIC_RRR);
  362. printk(KERN_CONT "%08x\n", status);
  363. break;
  364. default:
  365. printk(KERN_CONT "failed\n");
  366. }
  367. }
  368. }
  369. #endif
  370. /*
  371. * Kick the secondary to wake up.
  372. */
  373. static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int start_rip)
  374. {
  375. unsigned long send_status, accept_status = 0;
  376. int maxlvt, num_starts, j;
  377. Dprintk("Asserting INIT.\n");
  378. /*
  379. * Turn INIT on target chip
  380. */
  381. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  382. /*
  383. * Send IPI
  384. */
  385. apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
  386. | APIC_DM_INIT);
  387. Dprintk("Waiting for send to finish...\n");
  388. send_status = safe_apic_wait_icr_idle();
  389. mdelay(10);
  390. Dprintk("Deasserting INIT.\n");
  391. /* Target chip */
  392. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  393. /* Send IPI */
  394. apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
  395. Dprintk("Waiting for send to finish...\n");
  396. send_status = safe_apic_wait_icr_idle();
  397. mb();
  398. atomic_set(&init_deasserted, 1);
  399. num_starts = 2;
  400. /*
  401. * Run STARTUP IPI loop.
  402. */
  403. Dprintk("#startup loops: %d.\n", num_starts);
  404. maxlvt = lapic_get_maxlvt();
  405. for (j = 1; j <= num_starts; j++) {
  406. Dprintk("Sending STARTUP #%d.\n",j);
  407. apic_write(APIC_ESR, 0);
  408. apic_read(APIC_ESR);
  409. Dprintk("After apic_write.\n");
  410. /*
  411. * STARTUP IPI
  412. */
  413. /* Target chip */
  414. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  415. /* Boot on the stack */
  416. /* Kick the second */
  417. apic_write(APIC_ICR, APIC_DM_STARTUP | (start_rip >> 12));
  418. /*
  419. * Give the other CPU some time to accept the IPI.
  420. */
  421. udelay(300);
  422. Dprintk("Startup point 1.\n");
  423. Dprintk("Waiting for send to finish...\n");
  424. send_status = safe_apic_wait_icr_idle();
  425. /*
  426. * Give the other CPU some time to accept the IPI.
  427. */
  428. udelay(200);
  429. /*
  430. * Due to the Pentium erratum 3AP.
  431. */
  432. if (maxlvt > 3) {
  433. apic_write(APIC_ESR, 0);
  434. }
  435. accept_status = (apic_read(APIC_ESR) & 0xEF);
  436. if (send_status || accept_status)
  437. break;
  438. }
  439. Dprintk("After Startup.\n");
  440. if (send_status)
  441. printk(KERN_ERR "APIC never delivered???\n");
  442. if (accept_status)
  443. printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
  444. return (send_status | accept_status);
  445. }
  446. struct create_idle {
  447. struct work_struct work;
  448. struct task_struct *idle;
  449. struct completion done;
  450. int cpu;
  451. };
  452. static void __cpuinit do_fork_idle(struct work_struct *work)
  453. {
  454. struct create_idle *c_idle =
  455. container_of(work, struct create_idle, work);
  456. c_idle->idle = fork_idle(c_idle->cpu);
  457. complete(&c_idle->done);
  458. }
  459. /*
  460. * Boot one CPU.
  461. */
  462. static int __cpuinit do_boot_cpu(int cpu, int apicid)
  463. {
  464. unsigned long boot_error;
  465. int timeout;
  466. unsigned long start_rip;
  467. struct create_idle c_idle = {
  468. .work = __WORK_INITIALIZER(c_idle.work, do_fork_idle),
  469. .cpu = cpu,
  470. .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
  471. };
  472. /* allocate memory for gdts of secondary cpus. Hotplug is considered */
  473. if (!cpu_gdt_descr[cpu].address &&
  474. !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) {
  475. printk(KERN_ERR "Failed to allocate GDT for CPU %d\n", cpu);
  476. return -1;
  477. }
  478. /* Allocate node local memory for AP pdas */
  479. if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) {
  480. struct x8664_pda *newpda, *pda;
  481. int node = cpu_to_node(cpu);
  482. pda = cpu_pda(cpu);
  483. newpda = kmalloc_node(sizeof (struct x8664_pda), GFP_ATOMIC,
  484. node);
  485. if (newpda) {
  486. memcpy(newpda, pda, sizeof (struct x8664_pda));
  487. cpu_pda(cpu) = newpda;
  488. } else
  489. printk(KERN_ERR
  490. "Could not allocate node local PDA for CPU %d on node %d\n",
  491. cpu, node);
  492. }
  493. alternatives_smp_switch(1);
  494. c_idle.idle = get_idle_for_cpu(cpu);
  495. if (c_idle.idle) {
  496. c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
  497. (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
  498. init_idle(c_idle.idle, cpu);
  499. goto do_rest;
  500. }
  501. /*
  502. * During cold boot process, keventd thread is not spun up yet.
  503. * When we do cpu hot-add, we create idle threads on the fly, we should
  504. * not acquire any attributes from the calling context. Hence the clean
  505. * way to create kernel_threads() is to do that from keventd().
  506. * We do the current_is_keventd() due to the fact that ACPI notifier
  507. * was also queuing to keventd() and when the caller is already running
  508. * in context of keventd(), we would end up with locking up the keventd
  509. * thread.
  510. */
  511. if (!keventd_up() || current_is_keventd())
  512. c_idle.work.func(&c_idle.work);
  513. else {
  514. schedule_work(&c_idle.work);
  515. wait_for_completion(&c_idle.done);
  516. }
  517. if (IS_ERR(c_idle.idle)) {
  518. printk("failed fork for CPU %d\n", cpu);
  519. return PTR_ERR(c_idle.idle);
  520. }
  521. set_idle_for_cpu(cpu, c_idle.idle);
  522. do_rest:
  523. cpu_pda(cpu)->pcurrent = c_idle.idle;
  524. start_rip = setup_trampoline();
  525. init_rsp = c_idle.idle->thread.sp;
  526. load_sp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
  527. initial_code = start_secondary;
  528. clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
  529. printk(KERN_INFO "Booting processor %d/%d APIC 0x%x\n", cpu,
  530. cpus_weight(cpu_present_map),
  531. apicid);
  532. /*
  533. * This grunge runs the startup process for
  534. * the targeted processor.
  535. */
  536. atomic_set(&init_deasserted, 0);
  537. Dprintk("Setting warm reset code and vector.\n");
  538. CMOS_WRITE(0xa, 0xf);
  539. local_flush_tlb();
  540. Dprintk("1.\n");
  541. *((volatile unsigned short *) phys_to_virt(0x469)) = start_rip >> 4;
  542. Dprintk("2.\n");
  543. *((volatile unsigned short *) phys_to_virt(0x467)) = start_rip & 0xf;
  544. Dprintk("3.\n");
  545. /*
  546. * Be paranoid about clearing APIC errors.
  547. */
  548. apic_write(APIC_ESR, 0);
  549. apic_read(APIC_ESR);
  550. /*
  551. * Status is now clean
  552. */
  553. boot_error = 0;
  554. /*
  555. * Starting actual IPI sequence...
  556. */
  557. boot_error = wakeup_secondary_via_INIT(apicid, start_rip);
  558. if (!boot_error) {
  559. /*
  560. * allow APs to start initializing.
  561. */
  562. Dprintk("Before Callout %d.\n", cpu);
  563. cpu_set(cpu, cpu_callout_map);
  564. Dprintk("After Callout %d.\n", cpu);
  565. /*
  566. * Wait 5s total for a response
  567. */
  568. for (timeout = 0; timeout < 50000; timeout++) {
  569. if (cpu_isset(cpu, cpu_callin_map))
  570. break; /* It has booted */
  571. udelay(100);
  572. }
  573. if (cpu_isset(cpu, cpu_callin_map)) {
  574. /* number CPUs logically, starting from 1 (BSP is 0) */
  575. Dprintk("CPU has booted.\n");
  576. } else {
  577. boot_error = 1;
  578. if (*((volatile unsigned char *)phys_to_virt(SMP_TRAMPOLINE_BASE))
  579. == 0xA5)
  580. /* trampoline started but...? */
  581. printk("Stuck ??\n");
  582. else
  583. /* trampoline code not run */
  584. printk("Not responding.\n");
  585. #ifdef APIC_DEBUG
  586. inquire_remote_apic(apicid);
  587. #endif
  588. }
  589. }
  590. if (boot_error) {
  591. cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
  592. clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
  593. clear_node_cpumask(cpu); /* was set by numa_add_cpu */
  594. cpu_clear(cpu, cpu_present_map);
  595. cpu_clear(cpu, cpu_possible_map);
  596. per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
  597. return -EIO;
  598. }
  599. return 0;
  600. }
  601. cycles_t cacheflush_time;
  602. unsigned long cache_decay_ticks;
  603. /*
  604. * Cleanup possible dangling ends...
  605. */
  606. static __cpuinit void smp_cleanup_boot(void)
  607. {
  608. /*
  609. * Paranoid: Set warm reset code and vector here back
  610. * to default values.
  611. */
  612. CMOS_WRITE(0, 0xf);
  613. /*
  614. * Reset trampoline flag
  615. */
  616. *((volatile int *) phys_to_virt(0x467)) = 0;
  617. }
  618. /*
  619. * Fall back to non SMP mode after errors.
  620. *
  621. * RED-PEN audit/test this more. I bet there is more state messed up here.
  622. */
  623. static __init void disable_smp(void)
  624. {
  625. cpu_present_map = cpumask_of_cpu(0);
  626. cpu_possible_map = cpumask_of_cpu(0);
  627. if (smp_found_config)
  628. phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id);
  629. else
  630. phys_cpu_present_map = physid_mask_of_physid(0);
  631. cpu_set(0, per_cpu(cpu_sibling_map, 0));
  632. cpu_set(0, per_cpu(cpu_core_map, 0));
  633. }
  634. #ifdef CONFIG_HOTPLUG_CPU
  635. int additional_cpus __initdata = -1;
  636. /*
  637. * cpu_possible_map should be static, it cannot change as cpu's
  638. * are onlined, or offlined. The reason is per-cpu data-structures
  639. * are allocated by some modules at init time, and dont expect to
  640. * do this dynamically on cpu arrival/departure.
  641. * cpu_present_map on the other hand can change dynamically.
  642. * In case when cpu_hotplug is not compiled, then we resort to current
  643. * behaviour, which is cpu_possible == cpu_present.
  644. * - Ashok Raj
  645. *
  646. * Three ways to find out the number of additional hotplug CPUs:
  647. * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
  648. * - The user can overwrite it with additional_cpus=NUM
  649. * - Otherwise don't reserve additional CPUs.
  650. * We do this because additional CPUs waste a lot of memory.
  651. * -AK
  652. */
  653. __init void prefill_possible_map(void)
  654. {
  655. int i;
  656. int possible;
  657. if (additional_cpus == -1) {
  658. if (disabled_cpus > 0)
  659. additional_cpus = disabled_cpus;
  660. else
  661. additional_cpus = 0;
  662. }
  663. possible = num_processors + additional_cpus;
  664. if (possible > NR_CPUS)
  665. possible = NR_CPUS;
  666. printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
  667. possible,
  668. max_t(int, possible - num_processors, 0));
  669. for (i = 0; i < possible; i++)
  670. cpu_set(i, cpu_possible_map);
  671. }
  672. #endif
  673. /*
  674. * Various sanity checks.
  675. */
  676. static int __init smp_sanity_check(unsigned max_cpus)
  677. {
  678. if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
  679. printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
  680. hard_smp_processor_id());
  681. physid_set(hard_smp_processor_id(), phys_cpu_present_map);
  682. }
  683. /*
  684. * If we couldn't find an SMP configuration at boot time,
  685. * get out of here now!
  686. */
  687. if (!smp_found_config) {
  688. printk(KERN_NOTICE "SMP motherboard not detected.\n");
  689. disable_smp();
  690. if (APIC_init_uniprocessor())
  691. printk(KERN_NOTICE "Local APIC not detected."
  692. " Using dummy APIC emulation.\n");
  693. return -1;
  694. }
  695. /*
  696. * Should not be necessary because the MP table should list the boot
  697. * CPU too, but we do it for the sake of robustness anyway.
  698. */
  699. if (!physid_isset(boot_cpu_id, phys_cpu_present_map)) {
  700. printk(KERN_NOTICE "weird, boot CPU (#%d) not listed by the BIOS.\n",
  701. boot_cpu_id);
  702. physid_set(hard_smp_processor_id(), phys_cpu_present_map);
  703. }
  704. /*
  705. * If we couldn't find a local APIC, then get out of here now!
  706. */
  707. if (!cpu_has_apic) {
  708. printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
  709. boot_cpu_id);
  710. printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
  711. nr_ioapics = 0;
  712. return -1;
  713. }
  714. /*
  715. * If SMP should be disabled, then really disable it!
  716. */
  717. if (!max_cpus) {
  718. printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
  719. nr_ioapics = 0;
  720. return -1;
  721. }
  722. return 0;
  723. }
  724. /*
  725. * Copy data used in early init routines from the initial arrays to the
  726. * per cpu data areas. These arrays then become expendable and the
  727. * *_ptrs are zeroed indicating that the static arrays are gone.
  728. */
  729. void __init smp_set_apicids(void)
  730. {
  731. int cpu;
  732. for_each_possible_cpu(cpu) {
  733. if (per_cpu_offset(cpu)) {
  734. per_cpu(x86_cpu_to_apicid, cpu) =
  735. x86_cpu_to_apicid_init[cpu];
  736. #ifdef CONFIG_NUMA
  737. per_cpu(x86_cpu_to_node_map, cpu) =
  738. x86_cpu_to_node_map_init[cpu];
  739. #endif
  740. per_cpu(x86_bios_cpu_apicid, cpu) =
  741. x86_bios_cpu_apicid_init[cpu];
  742. }
  743. else
  744. printk(KERN_NOTICE "per_cpu_offset zero for cpu %d\n",
  745. cpu);
  746. }
  747. /* indicate the early static arrays are gone */
  748. x86_cpu_to_apicid_early_ptr = NULL;
  749. #ifdef CONFIG_NUMA
  750. x86_cpu_to_node_map_early_ptr = NULL;
  751. #endif
  752. x86_bios_cpu_apicid_early_ptr = NULL;
  753. }
  754. static void __init smp_cpu_index_default(void)
  755. {
  756. int i;
  757. struct cpuinfo_x86 *c;
  758. for_each_cpu_mask(i, cpu_possible_map) {
  759. c = &cpu_data(i);
  760. /* mark all to hotplug */
  761. c->cpu_index = NR_CPUS;
  762. }
  763. }
  764. /*
  765. * Prepare for SMP bootup. The MP table or ACPI has been read
  766. * earlier. Just do some sanity checking here and enable APIC mode.
  767. */
  768. void __init smp_prepare_cpus(unsigned int max_cpus)
  769. {
  770. nmi_watchdog_default();
  771. smp_cpu_index_default();
  772. current_cpu_data = boot_cpu_data;
  773. current_thread_info()->cpu = 0; /* needed? */
  774. smp_set_apicids();
  775. set_cpu_sibling_map(0);
  776. if (smp_sanity_check(max_cpus) < 0) {
  777. printk(KERN_INFO "SMP disabled\n");
  778. disable_smp();
  779. return;
  780. }
  781. /*
  782. * Switch from PIC to APIC mode.
  783. */
  784. setup_local_APIC();
  785. /*
  786. * Enable IO APIC before setting up error vector
  787. */
  788. if (!skip_ioapic_setup && nr_ioapics)
  789. enable_IO_APIC();
  790. end_local_APIC_setup();
  791. if (GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_id) {
  792. panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
  793. GET_APIC_ID(apic_read(APIC_ID)), boot_cpu_id);
  794. /* Or can we switch back to PIC here? */
  795. }
  796. /*
  797. * Now start the IO-APICs
  798. */
  799. if (!skip_ioapic_setup && nr_ioapics)
  800. setup_IO_APIC();
  801. else
  802. nr_ioapics = 0;
  803. /*
  804. * Set up local APIC timer on boot CPU.
  805. */
  806. setup_boot_clock();
  807. }
  808. /*
  809. * Early setup to make printk work.
  810. */
  811. void __init smp_prepare_boot_cpu(void)
  812. {
  813. int me = smp_processor_id();
  814. cpu_set(me, cpu_online_map);
  815. cpu_set(me, cpu_callout_map);
  816. per_cpu(cpu_state, me) = CPU_ONLINE;
  817. }
  818. /*
  819. * Entry point to boot a CPU.
  820. */
  821. int __cpuinit __cpu_up(unsigned int cpu)
  822. {
  823. int apicid = cpu_present_to_apicid(cpu);
  824. unsigned long flags;
  825. int err;
  826. WARN_ON(irqs_disabled());
  827. Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
  828. if (apicid == BAD_APICID || apicid == boot_cpu_id ||
  829. !physid_isset(apicid, phys_cpu_present_map)) {
  830. printk("__cpu_up: bad cpu %d\n", cpu);
  831. return -EINVAL;
  832. }
  833. /*
  834. * Already booted CPU?
  835. */
  836. if (cpu_isset(cpu, cpu_callin_map)) {
  837. Dprintk("do_boot_cpu %d Already started\n", cpu);
  838. return -ENOSYS;
  839. }
  840. /*
  841. * Save current MTRR state in case it was changed since early boot
  842. * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
  843. */
  844. mtrr_save_state();
  845. per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
  846. /* Boot it! */
  847. err = do_boot_cpu(cpu, apicid);
  848. if (err < 0) {
  849. Dprintk("do_boot_cpu failed %d\n", err);
  850. return err;
  851. }
  852. /* Unleash the CPU! */
  853. Dprintk("waiting for cpu %d\n", cpu);
  854. /*
  855. * Make sure and check TSC sync:
  856. */
  857. local_irq_save(flags);
  858. check_tsc_sync_source(cpu);
  859. local_irq_restore(flags);
  860. while (!cpu_isset(cpu, cpu_online_map))
  861. cpu_relax();
  862. err = 0;
  863. return err;
  864. }
  865. /*
  866. * Finish the SMP boot.
  867. */
  868. void __init smp_cpus_done(unsigned int max_cpus)
  869. {
  870. smp_cleanup_boot();
  871. setup_ioapic_dest();
  872. check_nmi_watchdog();
  873. }
  874. #ifdef CONFIG_HOTPLUG_CPU
  875. static void remove_siblinginfo(int cpu)
  876. {
  877. int sibling;
  878. struct cpuinfo_x86 *c = &cpu_data(cpu);
  879. for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
  880. cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
  881. /*
  882. * last thread sibling in this cpu core going down
  883. */
  884. if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
  885. cpu_data(sibling).booted_cores--;
  886. }
  887. for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
  888. cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
  889. cpus_clear(per_cpu(cpu_sibling_map, cpu));
  890. cpus_clear(per_cpu(cpu_core_map, cpu));
  891. c->phys_proc_id = 0;
  892. c->cpu_core_id = 0;
  893. cpu_clear(cpu, cpu_sibling_setup_map);
  894. }
  895. void remove_cpu_from_maps(void)
  896. {
  897. int cpu = smp_processor_id();
  898. cpu_clear(cpu, cpu_callout_map);
  899. cpu_clear(cpu, cpu_callin_map);
  900. clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
  901. clear_node_cpumask(cpu);
  902. }
  903. int __cpu_disable(void)
  904. {
  905. int cpu = smp_processor_id();
  906. /*
  907. * Perhaps use cpufreq to drop frequency, but that could go
  908. * into generic code.
  909. *
  910. * We won't take down the boot processor on i386 due to some
  911. * interrupts only being able to be serviced by the BSP.
  912. * Especially so if we're not using an IOAPIC -zwane
  913. */
  914. if (cpu == 0)
  915. return -EBUSY;
  916. if (nmi_watchdog == NMI_LOCAL_APIC)
  917. stop_apic_nmi_watchdog(NULL);
  918. clear_local_APIC();
  919. /*
  920. * HACK:
  921. * Allow any queued timer interrupts to get serviced
  922. * This is only a temporary solution until we cleanup
  923. * fixup_irqs as we do for IA64.
  924. */
  925. local_irq_enable();
  926. mdelay(1);
  927. local_irq_disable();
  928. remove_siblinginfo(cpu);
  929. spin_lock(&vector_lock);
  930. /* It's now safe to remove this processor from the online map */
  931. cpu_clear(cpu, cpu_online_map);
  932. spin_unlock(&vector_lock);
  933. remove_cpu_from_maps();
  934. fixup_irqs(cpu_online_map);
  935. return 0;
  936. }
  937. void __cpu_die(unsigned int cpu)
  938. {
  939. /* We don't do anything here: idle task is faking death itself. */
  940. unsigned int i;
  941. for (i = 0; i < 10; i++) {
  942. /* They ack this in play_dead by setting CPU_DEAD */
  943. if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
  944. printk ("CPU %d is now offline\n", cpu);
  945. if (1 == num_online_cpus())
  946. alternatives_smp_switch(0);
  947. return;
  948. }
  949. msleep(100);
  950. }
  951. printk(KERN_ERR "CPU %u didn't die...\n", cpu);
  952. }
  953. static __init int setup_additional_cpus(char *s)
  954. {
  955. return s && get_option(&s, &additional_cpus) ? 0 : -EINVAL;
  956. }
  957. early_param("additional_cpus", setup_additional_cpus);
  958. #else /* ... !CONFIG_HOTPLUG_CPU */
  959. int __cpu_disable(void)
  960. {
  961. return -ENOSYS;
  962. }
  963. void __cpu_die(unsigned int cpu)
  964. {
  965. /* We said "no" in __cpu_disable */
  966. BUG();
  967. }
  968. #endif /* CONFIG_HOTPLUG_CPU */