smpboot.c 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431
  1. /*
  2. * x86 SMP booting functions
  3. *
  4. * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
  5. * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
  6. *
  7. * Much of the core SMP work is based on previous work by Thomas Radke, to
  8. * whom a great many thanks are extended.
  9. *
  10. * Thanks to Intel for making available several different Pentium,
  11. * Pentium Pro and Pentium-II/Xeon MP machines.
  12. * Original development of Linux SMP code supported by Caldera.
  13. *
  14. * This code is released under the GNU General Public License version 2 or
  15. * later.
  16. *
  17. * Fixes
  18. * Felix Koop : NR_CPUS used properly
  19. * Jose Renau : Handle single CPU case.
  20. * Alan Cox : By repeated request 8) - Total BogoMIPS report.
  21. * Greg Wright : Fix for kernel stacks panic.
  22. * Erich Boleyn : MP v1.4 and additional changes.
  23. * Matthias Sattler : Changes for 2.1 kernel map.
  24. * Michel Lespinasse : Changes for 2.1 kernel map.
  25. * Michael Chastain : Change trampoline.S to gnu as.
  26. * Alan Cox : Dumb bug: 'B' step PPro's are fine
  27. * Ingo Molnar : Added APIC timers, based on code
  28. * from Jose Renau
  29. * Ingo Molnar : various cleanups and rewrites
  30. * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
  31. * Maciej W. Rozycki : Bits for genuine 82489DX APICs
  32. * Martin J. Bligh : Added support for multi-quad systems
  33. * Dave Jones : Report invalid combinations of Athlon CPUs.
  34. * Rusty Russell : Hacked into shape for new "hotplug" boot process. */
  35. #include <linux/module.h>
  36. #include <linux/config.h>
  37. #include <linux/init.h>
  38. #include <linux/kernel.h>
  39. #include <linux/mm.h>
  40. #include <linux/sched.h>
  41. #include <linux/kernel_stat.h>
  42. #include <linux/smp_lock.h>
  43. #include <linux/bootmem.h>
  44. #include <linux/notifier.h>
  45. #include <linux/cpu.h>
  46. #include <linux/percpu.h>
  47. #include <linux/delay.h>
  48. #include <linux/mc146818rtc.h>
  49. #include <asm/tlbflush.h>
  50. #include <asm/desc.h>
  51. #include <asm/arch_hooks.h>
  52. #include <mach_apic.h>
  53. #include <mach_wakecpu.h>
  54. #include <smpboot_hooks.h>
  55. /* Set if we find a B stepping CPU */
  56. static int __devinitdata smp_b_stepping;
  57. /* Number of siblings per CPU package */
  58. int smp_num_siblings = 1;
  59. #ifdef CONFIG_X86_HT
  60. EXPORT_SYMBOL(smp_num_siblings);
  61. #endif
  62. /* Package ID of each logical CPU */
  63. int phys_proc_id[NR_CPUS] __read_mostly = {[0 ... NR_CPUS-1] = BAD_APICID};
  64. /* Core ID of each logical CPU */
  65. int cpu_core_id[NR_CPUS] __read_mostly = {[0 ... NR_CPUS-1] = BAD_APICID};
  66. /* representing HT siblings of each logical CPU */
  67. cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
  68. EXPORT_SYMBOL(cpu_sibling_map);
  69. /* representing HT and core siblings of each logical CPU */
  70. cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
  71. EXPORT_SYMBOL(cpu_core_map);
  72. /* bitmap of online cpus */
  73. cpumask_t cpu_online_map __read_mostly;
  74. EXPORT_SYMBOL(cpu_online_map);
  75. cpumask_t cpu_callin_map;
  76. cpumask_t cpu_callout_map;
  77. EXPORT_SYMBOL(cpu_callout_map);
  78. cpumask_t cpu_possible_map;
  79. EXPORT_SYMBOL(cpu_possible_map);
  80. static cpumask_t smp_commenced_mask;
  81. /* TSC's upper 32 bits can't be written in eariler CPU (before prescott), there
  82. * is no way to resync one AP against BP. TBD: for prescott and above, we
  83. * should use IA64's algorithm
  84. */
  85. static int __devinitdata tsc_sync_disabled;
  86. /* Per CPU bogomips and other parameters */
  87. struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
  88. EXPORT_SYMBOL(cpu_data);
  89. u8 x86_cpu_to_apicid[NR_CPUS] __read_mostly =
  90. { [0 ... NR_CPUS-1] = 0xff };
  91. EXPORT_SYMBOL(x86_cpu_to_apicid);
  92. /*
  93. * Trampoline 80x86 program as an array.
  94. */
  95. extern unsigned char trampoline_data [];
  96. extern unsigned char trampoline_end [];
  97. static unsigned char *trampoline_base;
  98. static int trampoline_exec;
  99. static void map_cpu_to_logical_apicid(void);
  100. /* State of each CPU. */
  101. DEFINE_PER_CPU(int, cpu_state) = { 0 };
  102. /*
  103. * Currently trivial. Write the real->protected mode
  104. * bootstrap into the page concerned. The caller
  105. * has made sure it's suitably aligned.
  106. */
  107. static unsigned long __devinit setup_trampoline(void)
  108. {
  109. memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
  110. return virt_to_phys(trampoline_base);
  111. }
  112. /*
  113. * We are called very early to get the low memory for the
  114. * SMP bootup trampoline page.
  115. */
  116. void __init smp_alloc_memory(void)
  117. {
  118. trampoline_base = (void *) alloc_bootmem_low_pages(PAGE_SIZE);
  119. /*
  120. * Has to be in very low memory so we can execute
  121. * real-mode AP code.
  122. */
  123. if (__pa(trampoline_base) >= 0x9F000)
  124. BUG();
  125. /*
  126. * Make the SMP trampoline executable:
  127. */
  128. trampoline_exec = set_kernel_exec((unsigned long)trampoline_base, 1);
  129. }
  130. /*
  131. * The bootstrap kernel entry code has set these up. Save them for
  132. * a given CPU
  133. */
  134. static void __devinit smp_store_cpu_info(int id)
  135. {
  136. struct cpuinfo_x86 *c = cpu_data + id;
  137. *c = boot_cpu_data;
  138. if (id!=0)
  139. identify_cpu(c);
  140. /*
  141. * Mask B, Pentium, but not Pentium MMX
  142. */
  143. if (c->x86_vendor == X86_VENDOR_INTEL &&
  144. c->x86 == 5 &&
  145. c->x86_mask >= 1 && c->x86_mask <= 4 &&
  146. c->x86_model <= 3)
  147. /*
  148. * Remember we have B step Pentia with bugs
  149. */
  150. smp_b_stepping = 1;
  151. /*
  152. * Certain Athlons might work (for various values of 'work') in SMP
  153. * but they are not certified as MP capable.
  154. */
  155. if ((c->x86_vendor == X86_VENDOR_AMD) && (c->x86 == 6)) {
  156. /* Athlon 660/661 is valid. */
  157. if ((c->x86_model==6) && ((c->x86_mask==0) || (c->x86_mask==1)))
  158. goto valid_k7;
  159. /* Duron 670 is valid */
  160. if ((c->x86_model==7) && (c->x86_mask==0))
  161. goto valid_k7;
  162. /*
  163. * Athlon 662, Duron 671, and Athlon >model 7 have capability bit.
  164. * It's worth noting that the A5 stepping (662) of some Athlon XP's
  165. * have the MP bit set.
  166. * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for more.
  167. */
  168. if (((c->x86_model==6) && (c->x86_mask>=2)) ||
  169. ((c->x86_model==7) && (c->x86_mask>=1)) ||
  170. (c->x86_model> 7))
  171. if (cpu_has_mp)
  172. goto valid_k7;
  173. /* If we get here, it's not a certified SMP capable AMD system. */
  174. add_taint(TAINT_UNSAFE_SMP);
  175. }
  176. valid_k7:
  177. ;
  178. }
  179. /*
  180. * TSC synchronization.
  181. *
  182. * We first check whether all CPUs have their TSC's synchronized,
  183. * then we print a warning if not, and always resync.
  184. */
  185. static atomic_t tsc_start_flag = ATOMIC_INIT(0);
  186. static atomic_t tsc_count_start = ATOMIC_INIT(0);
  187. static atomic_t tsc_count_stop = ATOMIC_INIT(0);
  188. static unsigned long long tsc_values[NR_CPUS];
  189. #define NR_LOOPS 5
  190. static void __init synchronize_tsc_bp (void)
  191. {
  192. int i;
  193. unsigned long long t0;
  194. unsigned long long sum, avg;
  195. long long delta;
  196. unsigned int one_usec;
  197. int buggy = 0;
  198. printk(KERN_INFO "checking TSC synchronization across %u CPUs: ", num_booting_cpus());
  199. /* convert from kcyc/sec to cyc/usec */
  200. one_usec = cpu_khz / 1000;
  201. atomic_set(&tsc_start_flag, 1);
  202. wmb();
  203. /*
  204. * We loop a few times to get a primed instruction cache,
  205. * then the last pass is more or less synchronized and
  206. * the BP and APs set their cycle counters to zero all at
  207. * once. This reduces the chance of having random offsets
  208. * between the processors, and guarantees that the maximum
  209. * delay between the cycle counters is never bigger than
  210. * the latency of information-passing (cachelines) between
  211. * two CPUs.
  212. */
  213. for (i = 0; i < NR_LOOPS; i++) {
  214. /*
  215. * all APs synchronize but they loop on '== num_cpus'
  216. */
  217. while (atomic_read(&tsc_count_start) != num_booting_cpus()-1)
  218. mb();
  219. atomic_set(&tsc_count_stop, 0);
  220. wmb();
  221. /*
  222. * this lets the APs save their current TSC:
  223. */
  224. atomic_inc(&tsc_count_start);
  225. rdtscll(tsc_values[smp_processor_id()]);
  226. /*
  227. * We clear the TSC in the last loop:
  228. */
  229. if (i == NR_LOOPS-1)
  230. write_tsc(0, 0);
  231. /*
  232. * Wait for all APs to leave the synchronization point:
  233. */
  234. while (atomic_read(&tsc_count_stop) != num_booting_cpus()-1)
  235. mb();
  236. atomic_set(&tsc_count_start, 0);
  237. wmb();
  238. atomic_inc(&tsc_count_stop);
  239. }
  240. sum = 0;
  241. for (i = 0; i < NR_CPUS; i++) {
  242. if (cpu_isset(i, cpu_callout_map)) {
  243. t0 = tsc_values[i];
  244. sum += t0;
  245. }
  246. }
  247. avg = sum;
  248. do_div(avg, num_booting_cpus());
  249. sum = 0;
  250. for (i = 0; i < NR_CPUS; i++) {
  251. if (!cpu_isset(i, cpu_callout_map))
  252. continue;
  253. delta = tsc_values[i] - avg;
  254. if (delta < 0)
  255. delta = -delta;
  256. /*
  257. * We report bigger than 2 microseconds clock differences.
  258. */
  259. if (delta > 2*one_usec) {
  260. long realdelta;
  261. if (!buggy) {
  262. buggy = 1;
  263. printk("\n");
  264. }
  265. realdelta = delta;
  266. do_div(realdelta, one_usec);
  267. if (tsc_values[i] < avg)
  268. realdelta = -realdelta;
  269. printk(KERN_INFO "CPU#%d had %ld usecs TSC skew, fixed it up.\n", i, realdelta);
  270. }
  271. sum += delta;
  272. }
  273. if (!buggy)
  274. printk("passed.\n");
  275. }
  276. static void __init synchronize_tsc_ap (void)
  277. {
  278. int i;
  279. /*
  280. * Not every cpu is online at the time
  281. * this gets called, so we first wait for the BP to
  282. * finish SMP initialization:
  283. */
  284. while (!atomic_read(&tsc_start_flag)) mb();
  285. for (i = 0; i < NR_LOOPS; i++) {
  286. atomic_inc(&tsc_count_start);
  287. while (atomic_read(&tsc_count_start) != num_booting_cpus())
  288. mb();
  289. rdtscll(tsc_values[smp_processor_id()]);
  290. if (i == NR_LOOPS-1)
  291. write_tsc(0, 0);
  292. atomic_inc(&tsc_count_stop);
  293. while (atomic_read(&tsc_count_stop) != num_booting_cpus()) mb();
  294. }
  295. }
  296. #undef NR_LOOPS
  297. extern void calibrate_delay(void);
  298. static atomic_t init_deasserted;
  299. static void __devinit smp_callin(void)
  300. {
  301. int cpuid, phys_id;
  302. unsigned long timeout;
  303. /*
  304. * If waken up by an INIT in an 82489DX configuration
  305. * we may get here before an INIT-deassert IPI reaches
  306. * our local APIC. We have to wait for the IPI or we'll
  307. * lock up on an APIC access.
  308. */
  309. wait_for_init_deassert(&init_deasserted);
  310. /*
  311. * (This works even if the APIC is not enabled.)
  312. */
  313. phys_id = GET_APIC_ID(apic_read(APIC_ID));
  314. cpuid = smp_processor_id();
  315. if (cpu_isset(cpuid, cpu_callin_map)) {
  316. printk("huh, phys CPU#%d, CPU#%d already present??\n",
  317. phys_id, cpuid);
  318. BUG();
  319. }
  320. Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
  321. /*
  322. * STARTUP IPIs are fragile beasts as they might sometimes
  323. * trigger some glue motherboard logic. Complete APIC bus
  324. * silence for 1 second, this overestimates the time the
  325. * boot CPU is spending to send the up to 2 STARTUP IPIs
  326. * by a factor of two. This should be enough.
  327. */
  328. /*
  329. * Waiting 2s total for startup (udelay is not yet working)
  330. */
  331. timeout = jiffies + 2*HZ;
  332. while (time_before(jiffies, timeout)) {
  333. /*
  334. * Has the boot CPU finished it's STARTUP sequence?
  335. */
  336. if (cpu_isset(cpuid, cpu_callout_map))
  337. break;
  338. rep_nop();
  339. }
  340. if (!time_before(jiffies, timeout)) {
  341. printk("BUG: CPU%d started up but did not get a callout!\n",
  342. cpuid);
  343. BUG();
  344. }
  345. /*
  346. * the boot CPU has finished the init stage and is spinning
  347. * on callin_map until we finish. We are free to set up this
  348. * CPU, first the APIC. (this is probably redundant on most
  349. * boards)
  350. */
  351. Dprintk("CALLIN, before setup_local_APIC().\n");
  352. smp_callin_clear_local_apic();
  353. setup_local_APIC();
  354. map_cpu_to_logical_apicid();
  355. /*
  356. * Get our bogomips.
  357. */
  358. calibrate_delay();
  359. Dprintk("Stack at about %p\n",&cpuid);
  360. /*
  361. * Save our processor parameters
  362. */
  363. smp_store_cpu_info(cpuid);
  364. disable_APIC_timer();
  365. /*
  366. * Allow the master to continue.
  367. */
  368. cpu_set(cpuid, cpu_callin_map);
  369. /*
  370. * Synchronize the TSC with the BP
  371. */
  372. if (cpu_has_tsc && cpu_khz && !tsc_sync_disabled)
  373. synchronize_tsc_ap();
  374. }
  375. static int cpucount;
  376. /* representing cpus for which sibling maps can be computed */
  377. static cpumask_t cpu_sibling_setup_map;
  378. static inline void
  379. set_cpu_sibling_map(int cpu)
  380. {
  381. int i;
  382. struct cpuinfo_x86 *c = cpu_data;
  383. cpu_set(cpu, cpu_sibling_setup_map);
  384. if (smp_num_siblings > 1) {
  385. for_each_cpu_mask(i, cpu_sibling_setup_map) {
  386. if (phys_proc_id[cpu] == phys_proc_id[i] &&
  387. cpu_core_id[cpu] == cpu_core_id[i]) {
  388. cpu_set(i, cpu_sibling_map[cpu]);
  389. cpu_set(cpu, cpu_sibling_map[i]);
  390. cpu_set(i, cpu_core_map[cpu]);
  391. cpu_set(cpu, cpu_core_map[i]);
  392. }
  393. }
  394. } else {
  395. cpu_set(cpu, cpu_sibling_map[cpu]);
  396. }
  397. if (current_cpu_data.x86_max_cores == 1) {
  398. cpu_core_map[cpu] = cpu_sibling_map[cpu];
  399. c[cpu].booted_cores = 1;
  400. return;
  401. }
  402. for_each_cpu_mask(i, cpu_sibling_setup_map) {
  403. if (phys_proc_id[cpu] == phys_proc_id[i]) {
  404. cpu_set(i, cpu_core_map[cpu]);
  405. cpu_set(cpu, cpu_core_map[i]);
  406. /*
  407. * Does this new cpu bringup a new core?
  408. */
  409. if (cpus_weight(cpu_sibling_map[cpu]) == 1) {
  410. /*
  411. * for each core in package, increment
  412. * the booted_cores for this new cpu
  413. */
  414. if (first_cpu(cpu_sibling_map[i]) == i)
  415. c[cpu].booted_cores++;
  416. /*
  417. * increment the core count for all
  418. * the other cpus in this package
  419. */
  420. if (i != cpu)
  421. c[i].booted_cores++;
  422. } else if (i != cpu && !c[cpu].booted_cores)
  423. c[cpu].booted_cores = c[i].booted_cores;
  424. }
  425. }
  426. }
  427. /*
  428. * Activate a secondary processor.
  429. */
  430. static void __devinit start_secondary(void *unused)
  431. {
  432. /*
  433. * Dont put anything before smp_callin(), SMP
  434. * booting is too fragile that we want to limit the
  435. * things done here to the most necessary things.
  436. */
  437. cpu_init();
  438. preempt_disable();
  439. smp_callin();
  440. while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
  441. rep_nop();
  442. setup_secondary_APIC_clock();
  443. if (nmi_watchdog == NMI_IO_APIC) {
  444. disable_8259A_irq(0);
  445. enable_NMI_through_LVT0(NULL);
  446. enable_8259A_irq(0);
  447. }
  448. enable_APIC_timer();
  449. /*
  450. * low-memory mappings have been cleared, flush them from
  451. * the local TLBs too.
  452. */
  453. local_flush_tlb();
  454. /* This must be done before setting cpu_online_map */
  455. set_cpu_sibling_map(raw_smp_processor_id());
  456. wmb();
  457. /*
  458. * We need to hold call_lock, so there is no inconsistency
  459. * between the time smp_call_function() determines number of
  460. * IPI receipients, and the time when the determination is made
  461. * for which cpus receive the IPI. Holding this
  462. * lock helps us to not include this cpu in a currently in progress
  463. * smp_call_function().
  464. */
  465. lock_ipi_call_lock();
  466. cpu_set(smp_processor_id(), cpu_online_map);
  467. unlock_ipi_call_lock();
  468. per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
  469. /* We can take interrupts now: we're officially "up". */
  470. local_irq_enable();
  471. wmb();
  472. cpu_idle();
  473. }
  474. /*
  475. * Everything has been set up for the secondary
  476. * CPUs - they just need to reload everything
  477. * from the task structure
  478. * This function must not return.
  479. */
  480. void __devinit initialize_secondary(void)
  481. {
  482. /*
  483. * We don't actually need to load the full TSS,
  484. * basically just the stack pointer and the eip.
  485. */
  486. asm volatile(
  487. "movl %0,%%esp\n\t"
  488. "jmp *%1"
  489. :
  490. :"r" (current->thread.esp),"r" (current->thread.eip));
  491. }
  492. extern struct {
  493. void * esp;
  494. unsigned short ss;
  495. } stack_start;
  496. #ifdef CONFIG_NUMA
  497. /* which logical CPUs are on which nodes */
  498. cpumask_t node_2_cpu_mask[MAX_NUMNODES] __read_mostly =
  499. { [0 ... MAX_NUMNODES-1] = CPU_MASK_NONE };
  500. /* which node each logical CPU is on */
  501. int cpu_2_node[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
  502. EXPORT_SYMBOL(cpu_2_node);
  503. /* set up a mapping between cpu and node. */
  504. static inline void map_cpu_to_node(int cpu, int node)
  505. {
  506. printk("Mapping cpu %d to node %d\n", cpu, node);
  507. cpu_set(cpu, node_2_cpu_mask[node]);
  508. cpu_2_node[cpu] = node;
  509. }
  510. /* undo a mapping between cpu and node. */
  511. static inline void unmap_cpu_to_node(int cpu)
  512. {
  513. int node;
  514. printk("Unmapping cpu %d from all nodes\n", cpu);
  515. for (node = 0; node < MAX_NUMNODES; node ++)
  516. cpu_clear(cpu, node_2_cpu_mask[node]);
  517. cpu_2_node[cpu] = 0;
  518. }
  519. #else /* !CONFIG_NUMA */
  520. #define map_cpu_to_node(cpu, node) ({})
  521. #define unmap_cpu_to_node(cpu) ({})
  522. #endif /* CONFIG_NUMA */
  523. u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
  524. static void map_cpu_to_logical_apicid(void)
  525. {
  526. int cpu = smp_processor_id();
  527. int apicid = logical_smp_processor_id();
  528. cpu_2_logical_apicid[cpu] = apicid;
  529. map_cpu_to_node(cpu, apicid_to_node(apicid));
  530. }
  531. static void unmap_cpu_to_logical_apicid(int cpu)
  532. {
  533. cpu_2_logical_apicid[cpu] = BAD_APICID;
  534. unmap_cpu_to_node(cpu);
  535. }
  536. #if APIC_DEBUG
  537. static inline void __inquire_remote_apic(int apicid)
  538. {
  539. int i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
  540. char *names[] = { "ID", "VERSION", "SPIV" };
  541. int timeout, status;
  542. printk("Inquiring remote APIC #%d...\n", apicid);
  543. for (i = 0; i < ARRAY_SIZE(regs); i++) {
  544. printk("... APIC #%d %s: ", apicid, names[i]);
  545. /*
  546. * Wait for idle.
  547. */
  548. apic_wait_icr_idle();
  549. apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
  550. apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
  551. timeout = 0;
  552. do {
  553. udelay(100);
  554. status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
  555. } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
  556. switch (status) {
  557. case APIC_ICR_RR_VALID:
  558. status = apic_read(APIC_RRR);
  559. printk("%08x\n", status);
  560. break;
  561. default:
  562. printk("failed\n");
  563. }
  564. }
  565. }
  566. #endif
  567. #ifdef WAKE_SECONDARY_VIA_NMI
  568. /*
  569. * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
  570. * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
  571. * won't ... remember to clear down the APIC, etc later.
  572. */
  573. static int __devinit
  574. wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
  575. {
  576. unsigned long send_status = 0, accept_status = 0;
  577. int timeout, maxlvt;
  578. /* Target chip */
  579. apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
  580. /* Boot on the stack */
  581. /* Kick the second */
  582. apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
  583. Dprintk("Waiting for send to finish...\n");
  584. timeout = 0;
  585. do {
  586. Dprintk("+");
  587. udelay(100);
  588. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  589. } while (send_status && (timeout++ < 1000));
  590. /*
  591. * Give the other CPU some time to accept the IPI.
  592. */
  593. udelay(200);
  594. /*
  595. * Due to the Pentium erratum 3AP.
  596. */
  597. maxlvt = get_maxlvt();
  598. if (maxlvt > 3) {
  599. apic_read_around(APIC_SPIV);
  600. apic_write(APIC_ESR, 0);
  601. }
  602. accept_status = (apic_read(APIC_ESR) & 0xEF);
  603. Dprintk("NMI sent.\n");
  604. if (send_status)
  605. printk("APIC never delivered???\n");
  606. if (accept_status)
  607. printk("APIC delivery error (%lx).\n", accept_status);
  608. return (send_status | accept_status);
  609. }
  610. #endif /* WAKE_SECONDARY_VIA_NMI */
  611. #ifdef WAKE_SECONDARY_VIA_INIT
  612. static int __devinit
  613. wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
  614. {
  615. unsigned long send_status = 0, accept_status = 0;
  616. int maxlvt, timeout, num_starts, j;
  617. /*
  618. * Be paranoid about clearing APIC errors.
  619. */
  620. if (APIC_INTEGRATED(apic_version[phys_apicid])) {
  621. apic_read_around(APIC_SPIV);
  622. apic_write(APIC_ESR, 0);
  623. apic_read(APIC_ESR);
  624. }
  625. Dprintk("Asserting INIT.\n");
  626. /*
  627. * Turn INIT on target chip
  628. */
  629. apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  630. /*
  631. * Send IPI
  632. */
  633. apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
  634. | APIC_DM_INIT);
  635. Dprintk("Waiting for send to finish...\n");
  636. timeout = 0;
  637. do {
  638. Dprintk("+");
  639. udelay(100);
  640. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  641. } while (send_status && (timeout++ < 1000));
  642. mdelay(10);
  643. Dprintk("Deasserting INIT.\n");
  644. /* Target chip */
  645. apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  646. /* Send IPI */
  647. apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
  648. Dprintk("Waiting for send to finish...\n");
  649. timeout = 0;
  650. do {
  651. Dprintk("+");
  652. udelay(100);
  653. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  654. } while (send_status && (timeout++ < 1000));
  655. atomic_set(&init_deasserted, 1);
  656. /*
  657. * Should we send STARTUP IPIs ?
  658. *
  659. * Determine this based on the APIC version.
  660. * If we don't have an integrated APIC, don't send the STARTUP IPIs.
  661. */
  662. if (APIC_INTEGRATED(apic_version[phys_apicid]))
  663. num_starts = 2;
  664. else
  665. num_starts = 0;
  666. /*
  667. * Run STARTUP IPI loop.
  668. */
  669. Dprintk("#startup loops: %d.\n", num_starts);
  670. maxlvt = get_maxlvt();
  671. for (j = 1; j <= num_starts; j++) {
  672. Dprintk("Sending STARTUP #%d.\n",j);
  673. apic_read_around(APIC_SPIV);
  674. apic_write(APIC_ESR, 0);
  675. apic_read(APIC_ESR);
  676. Dprintk("After apic_write.\n");
  677. /*
  678. * STARTUP IPI
  679. */
  680. /* Target chip */
  681. apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
  682. /* Boot on the stack */
  683. /* Kick the second */
  684. apic_write_around(APIC_ICR, APIC_DM_STARTUP
  685. | (start_eip >> 12));
  686. /*
  687. * Give the other CPU some time to accept the IPI.
  688. */
  689. udelay(300);
  690. Dprintk("Startup point 1.\n");
  691. Dprintk("Waiting for send to finish...\n");
  692. timeout = 0;
  693. do {
  694. Dprintk("+");
  695. udelay(100);
  696. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  697. } while (send_status && (timeout++ < 1000));
  698. /*
  699. * Give the other CPU some time to accept the IPI.
  700. */
  701. udelay(200);
  702. /*
  703. * Due to the Pentium erratum 3AP.
  704. */
  705. if (maxlvt > 3) {
  706. apic_read_around(APIC_SPIV);
  707. apic_write(APIC_ESR, 0);
  708. }
  709. accept_status = (apic_read(APIC_ESR) & 0xEF);
  710. if (send_status || accept_status)
  711. break;
  712. }
  713. Dprintk("After Startup.\n");
  714. if (send_status)
  715. printk("APIC never delivered???\n");
  716. if (accept_status)
  717. printk("APIC delivery error (%lx).\n", accept_status);
  718. return (send_status | accept_status);
  719. }
  720. #endif /* WAKE_SECONDARY_VIA_INIT */
  721. extern cpumask_t cpu_initialized;
  722. static inline int alloc_cpu_id(void)
  723. {
  724. cpumask_t tmp_map;
  725. int cpu;
  726. cpus_complement(tmp_map, cpu_present_map);
  727. cpu = first_cpu(tmp_map);
  728. if (cpu >= NR_CPUS)
  729. return -ENODEV;
  730. return cpu;
  731. }
  732. #ifdef CONFIG_HOTPLUG_CPU
  733. static struct task_struct * __devinitdata cpu_idle_tasks[NR_CPUS];
  734. static inline struct task_struct * alloc_idle_task(int cpu)
  735. {
  736. struct task_struct *idle;
  737. if ((idle = cpu_idle_tasks[cpu]) != NULL) {
  738. /* initialize thread_struct. we really want to avoid destroy
  739. * idle tread
  740. */
  741. idle->thread.esp = (unsigned long)task_pt_regs(idle);
  742. init_idle(idle, cpu);
  743. return idle;
  744. }
  745. idle = fork_idle(cpu);
  746. if (!IS_ERR(idle))
  747. cpu_idle_tasks[cpu] = idle;
  748. return idle;
  749. }
  750. #else
  751. #define alloc_idle_task(cpu) fork_idle(cpu)
  752. #endif
  753. static int __devinit do_boot_cpu(int apicid, int cpu)
  754. /*
  755. * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
  756. * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
  757. * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
  758. */
  759. {
  760. struct task_struct *idle;
  761. unsigned long boot_error;
  762. int timeout;
  763. unsigned long start_eip;
  764. unsigned short nmi_high = 0, nmi_low = 0;
  765. ++cpucount;
  766. /*
  767. * We can't use kernel_thread since we must avoid to
  768. * reschedule the child.
  769. */
  770. idle = alloc_idle_task(cpu);
  771. if (IS_ERR(idle))
  772. panic("failed fork for CPU %d", cpu);
  773. idle->thread.eip = (unsigned long) start_secondary;
  774. /* start_eip had better be page-aligned! */
  775. start_eip = setup_trampoline();
  776. /* So we see what's up */
  777. printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip);
  778. /* Stack for startup_32 can be just as for start_secondary onwards */
  779. stack_start.esp = (void *) idle->thread.esp;
  780. irq_ctx_init(cpu);
  781. /*
  782. * This grunge runs the startup process for
  783. * the targeted processor.
  784. */
  785. atomic_set(&init_deasserted, 0);
  786. Dprintk("Setting warm reset code and vector.\n");
  787. store_NMI_vector(&nmi_high, &nmi_low);
  788. smpboot_setup_warm_reset_vector(start_eip);
  789. /*
  790. * Starting actual IPI sequence...
  791. */
  792. boot_error = wakeup_secondary_cpu(apicid, start_eip);
  793. if (!boot_error) {
  794. /*
  795. * allow APs to start initializing.
  796. */
  797. Dprintk("Before Callout %d.\n", cpu);
  798. cpu_set(cpu, cpu_callout_map);
  799. Dprintk("After Callout %d.\n", cpu);
  800. /*
  801. * Wait 5s total for a response
  802. */
  803. for (timeout = 0; timeout < 50000; timeout++) {
  804. if (cpu_isset(cpu, cpu_callin_map))
  805. break; /* It has booted */
  806. udelay(100);
  807. }
  808. if (cpu_isset(cpu, cpu_callin_map)) {
  809. /* number CPUs logically, starting from 1 (BSP is 0) */
  810. Dprintk("OK.\n");
  811. printk("CPU%d: ", cpu);
  812. print_cpu_info(&cpu_data[cpu]);
  813. Dprintk("CPU has booted.\n");
  814. } else {
  815. boot_error= 1;
  816. if (*((volatile unsigned char *)trampoline_base)
  817. == 0xA5)
  818. /* trampoline started but...? */
  819. printk("Stuck ??\n");
  820. else
  821. /* trampoline code not run */
  822. printk("Not responding.\n");
  823. inquire_remote_apic(apicid);
  824. }
  825. }
  826. if (boot_error) {
  827. /* Try to put things back the way they were before ... */
  828. unmap_cpu_to_logical_apicid(cpu);
  829. cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
  830. cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
  831. cpucount--;
  832. } else {
  833. x86_cpu_to_apicid[cpu] = apicid;
  834. cpu_set(cpu, cpu_present_map);
  835. }
  836. /* mark "stuck" area as not stuck */
  837. *((volatile unsigned long *)trampoline_base) = 0;
  838. return boot_error;
  839. }
  840. #ifdef CONFIG_HOTPLUG_CPU
  841. void cpu_exit_clear(void)
  842. {
  843. int cpu = raw_smp_processor_id();
  844. idle_task_exit();
  845. cpucount --;
  846. cpu_uninit();
  847. irq_ctx_exit(cpu);
  848. cpu_clear(cpu, cpu_callout_map);
  849. cpu_clear(cpu, cpu_callin_map);
  850. cpu_clear(cpu, cpu_present_map);
  851. cpu_clear(cpu, smp_commenced_mask);
  852. unmap_cpu_to_logical_apicid(cpu);
  853. }
  854. struct warm_boot_cpu_info {
  855. struct completion *complete;
  856. int apicid;
  857. int cpu;
  858. };
  859. static void __devinit do_warm_boot_cpu(void *p)
  860. {
  861. struct warm_boot_cpu_info *info = p;
  862. do_boot_cpu(info->apicid, info->cpu);
  863. complete(info->complete);
  864. }
  865. int __devinit smp_prepare_cpu(int cpu)
  866. {
  867. DECLARE_COMPLETION(done);
  868. struct warm_boot_cpu_info info;
  869. struct work_struct task;
  870. int apicid, ret;
  871. lock_cpu_hotplug();
  872. apicid = x86_cpu_to_apicid[cpu];
  873. if (apicid == BAD_APICID) {
  874. ret = -ENODEV;
  875. goto exit;
  876. }
  877. info.complete = &done;
  878. info.apicid = apicid;
  879. info.cpu = cpu;
  880. INIT_WORK(&task, do_warm_boot_cpu, &info);
  881. tsc_sync_disabled = 1;
  882. /* init low mem mapping */
  883. clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS,
  884. KERNEL_PGD_PTRS);
  885. flush_tlb_all();
  886. schedule_work(&task);
  887. wait_for_completion(&done);
  888. tsc_sync_disabled = 0;
  889. zap_low_mappings();
  890. ret = 0;
  891. exit:
  892. unlock_cpu_hotplug();
  893. return ret;
  894. }
  895. #endif
  896. static void smp_tune_scheduling (void)
  897. {
  898. unsigned long cachesize; /* kB */
  899. unsigned long bandwidth = 350; /* MB/s */
  900. /*
  901. * Rough estimation for SMP scheduling, this is the number of
  902. * cycles it takes for a fully memory-limited process to flush
  903. * the SMP-local cache.
  904. *
  905. * (For a P5 this pretty much means we will choose another idle
  906. * CPU almost always at wakeup time (this is due to the small
  907. * L1 cache), on PIIs it's around 50-100 usecs, depending on
  908. * the cache size)
  909. */
  910. if (!cpu_khz) {
  911. /*
  912. * this basically disables processor-affinity
  913. * scheduling on SMP without a TSC.
  914. */
  915. return;
  916. } else {
  917. cachesize = boot_cpu_data.x86_cache_size;
  918. if (cachesize == -1) {
  919. cachesize = 16; /* Pentiums, 2x8kB cache */
  920. bandwidth = 100;
  921. }
  922. max_cache_size = cachesize * 1024;
  923. }
  924. }
  925. /*
  926. * Cycle through the processors sending APIC IPIs to boot each.
  927. */
  928. static int boot_cpu_logical_apicid;
  929. /* Where the IO area was mapped on multiquad, always 0 otherwise */
  930. void *xquad_portio;
  931. #ifdef CONFIG_X86_NUMAQ
  932. EXPORT_SYMBOL(xquad_portio);
  933. #endif
  934. static void __init smp_boot_cpus(unsigned int max_cpus)
  935. {
  936. int apicid, cpu, bit, kicked;
  937. unsigned long bogosum = 0;
  938. /*
  939. * Setup boot CPU information
  940. */
  941. smp_store_cpu_info(0); /* Final full version of the data */
  942. printk("CPU%d: ", 0);
  943. print_cpu_info(&cpu_data[0]);
  944. boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
  945. boot_cpu_logical_apicid = logical_smp_processor_id();
  946. x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;
  947. current_thread_info()->cpu = 0;
  948. smp_tune_scheduling();
  949. set_cpu_sibling_map(0);
  950. /*
  951. * If we couldn't find an SMP configuration at boot time,
  952. * get out of here now!
  953. */
  954. if (!smp_found_config && !acpi_lapic) {
  955. printk(KERN_NOTICE "SMP motherboard not detected.\n");
  956. smpboot_clear_io_apic_irqs();
  957. phys_cpu_present_map = physid_mask_of_physid(0);
  958. if (APIC_init_uniprocessor())
  959. printk(KERN_NOTICE "Local APIC not detected."
  960. " Using dummy APIC emulation.\n");
  961. map_cpu_to_logical_apicid();
  962. cpu_set(0, cpu_sibling_map[0]);
  963. cpu_set(0, cpu_core_map[0]);
  964. return;
  965. }
  966. /*
  967. * Should not be necessary because the MP table should list the boot
  968. * CPU too, but we do it for the sake of robustness anyway.
  969. * Makes no sense to do this check in clustered apic mode, so skip it
  970. */
  971. if (!check_phys_apicid_present(boot_cpu_physical_apicid)) {
  972. printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
  973. boot_cpu_physical_apicid);
  974. physid_set(hard_smp_processor_id(), phys_cpu_present_map);
  975. }
  976. /*
  977. * If we couldn't find a local APIC, then get out of here now!
  978. */
  979. if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && !cpu_has_apic) {
  980. printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
  981. boot_cpu_physical_apicid);
  982. printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
  983. smpboot_clear_io_apic_irqs();
  984. phys_cpu_present_map = physid_mask_of_physid(0);
  985. cpu_set(0, cpu_sibling_map[0]);
  986. cpu_set(0, cpu_core_map[0]);
  987. return;
  988. }
  989. verify_local_APIC();
  990. /*
  991. * If SMP should be disabled, then really disable it!
  992. */
  993. if (!max_cpus) {
  994. smp_found_config = 0;
  995. printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
  996. smpboot_clear_io_apic_irqs();
  997. phys_cpu_present_map = physid_mask_of_physid(0);
  998. cpu_set(0, cpu_sibling_map[0]);
  999. cpu_set(0, cpu_core_map[0]);
  1000. return;
  1001. }
  1002. connect_bsp_APIC();
  1003. setup_local_APIC();
  1004. map_cpu_to_logical_apicid();
  1005. setup_portio_remap();
  1006. /*
  1007. * Scan the CPU present map and fire up the other CPUs via do_boot_cpu
  1008. *
  1009. * In clustered apic mode, phys_cpu_present_map is a constructed thus:
  1010. * bits 0-3 are quad0, 4-7 are quad1, etc. A perverse twist on the
  1011. * clustered apic ID.
  1012. */
  1013. Dprintk("CPU present map: %lx\n", physids_coerce(phys_cpu_present_map));
  1014. kicked = 1;
  1015. for (bit = 0; kicked < NR_CPUS && bit < MAX_APICS; bit++) {
  1016. apicid = cpu_present_to_apicid(bit);
  1017. /*
  1018. * Don't even attempt to start the boot CPU!
  1019. */
  1020. if ((apicid == boot_cpu_apicid) || (apicid == BAD_APICID))
  1021. continue;
  1022. if (!check_apicid_present(bit))
  1023. continue;
  1024. if (max_cpus <= cpucount+1)
  1025. continue;
  1026. if (((cpu = alloc_cpu_id()) <= 0) || do_boot_cpu(apicid, cpu))
  1027. printk("CPU #%d not responding - cannot use it.\n",
  1028. apicid);
  1029. else
  1030. ++kicked;
  1031. }
  1032. /*
  1033. * Cleanup possible dangling ends...
  1034. */
  1035. smpboot_restore_warm_reset_vector();
  1036. /*
  1037. * Allow the user to impress friends.
  1038. */
  1039. Dprintk("Before bogomips.\n");
  1040. for (cpu = 0; cpu < NR_CPUS; cpu++)
  1041. if (cpu_isset(cpu, cpu_callout_map))
  1042. bogosum += cpu_data[cpu].loops_per_jiffy;
  1043. printk(KERN_INFO
  1044. "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
  1045. cpucount+1,
  1046. bogosum/(500000/HZ),
  1047. (bogosum/(5000/HZ))%100);
  1048. Dprintk("Before bogocount - setting activated=1.\n");
  1049. if (smp_b_stepping)
  1050. printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");
  1051. /*
  1052. * Don't taint if we are running SMP kernel on a single non-MP
  1053. * approved Athlon
  1054. */
  1055. if (tainted & TAINT_UNSAFE_SMP) {
  1056. if (cpucount)
  1057. printk (KERN_INFO "WARNING: This combination of AMD processors is not suitable for SMP.\n");
  1058. else
  1059. tainted &= ~TAINT_UNSAFE_SMP;
  1060. }
  1061. Dprintk("Boot done.\n");
  1062. /*
  1063. * construct cpu_sibling_map[], so that we can tell sibling CPUs
  1064. * efficiently.
  1065. */
  1066. for (cpu = 0; cpu < NR_CPUS; cpu++) {
  1067. cpus_clear(cpu_sibling_map[cpu]);
  1068. cpus_clear(cpu_core_map[cpu]);
  1069. }
  1070. cpu_set(0, cpu_sibling_map[0]);
  1071. cpu_set(0, cpu_core_map[0]);
  1072. smpboot_setup_io_apic();
  1073. setup_boot_APIC_clock();
  1074. /*
  1075. * Synchronize the TSC with the AP
  1076. */
  1077. if (cpu_has_tsc && cpucount && cpu_khz)
  1078. synchronize_tsc_bp();
  1079. }
  1080. /* These are wrappers to interface to the new boot process. Someone
  1081. who understands all this stuff should rewrite it properly. --RR 15/Jul/02 */
  1082. void __init smp_prepare_cpus(unsigned int max_cpus)
  1083. {
  1084. smp_commenced_mask = cpumask_of_cpu(0);
  1085. cpu_callin_map = cpumask_of_cpu(0);
  1086. mb();
  1087. smp_boot_cpus(max_cpus);
  1088. }
  1089. void __devinit smp_prepare_boot_cpu(void)
  1090. {
  1091. cpu_set(smp_processor_id(), cpu_online_map);
  1092. cpu_set(smp_processor_id(), cpu_callout_map);
  1093. cpu_set(smp_processor_id(), cpu_present_map);
  1094. cpu_set(smp_processor_id(), cpu_possible_map);
  1095. per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
  1096. }
  1097. #ifdef CONFIG_HOTPLUG_CPU
  1098. static void
  1099. remove_siblinginfo(int cpu)
  1100. {
  1101. int sibling;
  1102. struct cpuinfo_x86 *c = cpu_data;
  1103. for_each_cpu_mask(sibling, cpu_core_map[cpu]) {
  1104. cpu_clear(cpu, cpu_core_map[sibling]);
  1105. /*
  1106. * last thread sibling in this cpu core going down
  1107. */
  1108. if (cpus_weight(cpu_sibling_map[cpu]) == 1)
  1109. c[sibling].booted_cores--;
  1110. }
  1111. for_each_cpu_mask(sibling, cpu_sibling_map[cpu])
  1112. cpu_clear(cpu, cpu_sibling_map[sibling]);
  1113. cpus_clear(cpu_sibling_map[cpu]);
  1114. cpus_clear(cpu_core_map[cpu]);
  1115. phys_proc_id[cpu] = BAD_APICID;
  1116. cpu_core_id[cpu] = BAD_APICID;
  1117. cpu_clear(cpu, cpu_sibling_setup_map);
  1118. }
  1119. int __cpu_disable(void)
  1120. {
  1121. cpumask_t map = cpu_online_map;
  1122. int cpu = smp_processor_id();
  1123. /*
  1124. * Perhaps use cpufreq to drop frequency, but that could go
  1125. * into generic code.
  1126. *
  1127. * We won't take down the boot processor on i386 due to some
  1128. * interrupts only being able to be serviced by the BSP.
  1129. * Especially so if we're not using an IOAPIC -zwane
  1130. */
  1131. if (cpu == 0)
  1132. return -EBUSY;
  1133. clear_local_APIC();
  1134. /* Allow any queued timer interrupts to get serviced */
  1135. local_irq_enable();
  1136. mdelay(1);
  1137. local_irq_disable();
  1138. remove_siblinginfo(cpu);
  1139. cpu_clear(cpu, map);
  1140. fixup_irqs(map);
  1141. /* It's now safe to remove this processor from the online map */
  1142. cpu_clear(cpu, cpu_online_map);
  1143. return 0;
  1144. }
  1145. void __cpu_die(unsigned int cpu)
  1146. {
  1147. /* We don't do anything here: idle task is faking death itself. */
  1148. unsigned int i;
  1149. for (i = 0; i < 10; i++) {
  1150. /* They ack this in play_dead by setting CPU_DEAD */
  1151. if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
  1152. printk ("CPU %d is now offline\n", cpu);
  1153. return;
  1154. }
  1155. msleep(100);
  1156. }
  1157. printk(KERN_ERR "CPU %u didn't die...\n", cpu);
  1158. }
  1159. #else /* ... !CONFIG_HOTPLUG_CPU */
  1160. int __cpu_disable(void)
  1161. {
  1162. return -ENOSYS;
  1163. }
  1164. void __cpu_die(unsigned int cpu)
  1165. {
  1166. /* We said "no" in __cpu_disable */
  1167. BUG();
  1168. }
  1169. #endif /* CONFIG_HOTPLUG_CPU */
  1170. int __devinit __cpu_up(unsigned int cpu)
  1171. {
  1172. /* In case one didn't come up */
  1173. if (!cpu_isset(cpu, cpu_callin_map)) {
  1174. printk(KERN_DEBUG "skipping cpu%d, didn't come online\n", cpu);
  1175. local_irq_enable();
  1176. return -EIO;
  1177. }
  1178. local_irq_enable();
  1179. per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
  1180. /* Unleash the CPU! */
  1181. cpu_set(cpu, smp_commenced_mask);
  1182. while (!cpu_isset(cpu, cpu_online_map))
  1183. mb();
  1184. return 0;
  1185. }
  1186. void __init smp_cpus_done(unsigned int max_cpus)
  1187. {
  1188. #ifdef CONFIG_X86_IO_APIC
  1189. setup_ioapic_dest();
  1190. #endif
  1191. zap_low_mappings();
  1192. #ifndef CONFIG_HOTPLUG_CPU
  1193. /*
  1194. * Disable executability of the SMP trampoline:
  1195. */
  1196. set_kernel_exec((unsigned long)trampoline_base, trampoline_exec);
  1197. #endif
  1198. }
  1199. void __init smp_intr_init(void)
  1200. {
  1201. /*
  1202. * IRQ0 must be given a fixed assignment and initialized,
  1203. * because it's used before the IO-APIC is set up.
  1204. */
  1205. set_intr_gate(FIRST_DEVICE_VECTOR, interrupt[0]);
  1206. /*
  1207. * The reschedule interrupt is a CPU-to-CPU reschedule-helper
  1208. * IPI, driven by wakeup.
  1209. */
  1210. set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
  1211. /* IPI for invalidation */
  1212. set_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
  1213. /* IPI for generic function call */
  1214. set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
  1215. }