apic.c 30 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195
  1. /*
  2. * Local APIC handling, local APIC timers
  3. *
  4. * (c) 1999, 2000 Ingo Molnar <mingo@redhat.com>
  5. *
  6. * Fixes
  7. * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
  8. * thanks to Eric Gilmore
  9. * and Rolf G. Tews
  10. * for testing these extensively.
  11. * Maciej W. Rozycki : Various updates and fixes.
  12. * Mikael Pettersson : Power Management for UP-APIC.
  13. * Pavel Machek and
  14. * Mikael Pettersson : PM converted to driver model.
  15. */
  16. #include <linux/config.h>
  17. #include <linux/init.h>
  18. #include <linux/mm.h>
  19. #include <linux/delay.h>
  20. #include <linux/bootmem.h>
  21. #include <linux/smp_lock.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/mc146818rtc.h>
  24. #include <linux/kernel_stat.h>
  25. #include <linux/sysdev.h>
  26. #include <linux/module.h>
  27. #include <asm/atomic.h>
  28. #include <asm/smp.h>
  29. #include <asm/mtrr.h>
  30. #include <asm/mpspec.h>
  31. #include <asm/pgalloc.h>
  32. #include <asm/mach_apic.h>
  33. #include <asm/nmi.h>
  34. #include <asm/idle.h>
  35. #include <asm/proto.h>
  36. #include <asm/timex.h>
  37. int apic_verbosity;
  38. int apic_runs_main_timer;
  39. int apic_calibrate_pmtmr __initdata;
  40. int disable_apic_timer __initdata;
  41. /*
  42. * cpu_mask that denotes the CPUs that needs timer interrupt coming in as
  43. * IPIs in place of local APIC timers
  44. */
  45. static cpumask_t timer_interrupt_broadcast_ipi_mask;
  46. /* Using APIC to generate smp_local_timer_interrupt? */
  47. int using_apic_timer __read_mostly = 0;
  48. static void apic_pm_activate(void);
  49. void enable_NMI_through_LVT0 (void * dummy)
  50. {
  51. unsigned int v;
  52. v = APIC_DM_NMI; /* unmask and set to NMI */
  53. apic_write(APIC_LVT0, v);
  54. }
  55. int get_maxlvt(void)
  56. {
  57. unsigned int v, maxlvt;
  58. v = apic_read(APIC_LVR);
  59. maxlvt = GET_APIC_MAXLVT(v);
  60. return maxlvt;
  61. }
  62. /*
  63. * 'what should we do if we get a hw irq event on an illegal vector'.
  64. * each architecture has to answer this themselves.
  65. */
  66. void ack_bad_irq(unsigned int irq)
  67. {
  68. printk("unexpected IRQ trap at vector %02x\n", irq);
  69. /*
  70. * Currently unexpected vectors happen only on SMP and APIC.
  71. * We _must_ ack these because every local APIC has only N
  72. * irq slots per priority level, and a 'hanging, unacked' IRQ
  73. * holds up an irq slot - in excessive cases (when multiple
  74. * unexpected vectors occur) that might lock up the APIC
  75. * completely.
  76. * But don't ack when the APIC is disabled. -AK
  77. */
  78. if (!disable_apic)
  79. ack_APIC_irq();
  80. }
  81. void clear_local_APIC(void)
  82. {
  83. int maxlvt;
  84. unsigned int v;
  85. maxlvt = get_maxlvt();
  86. /*
  87. * Masking an LVT entry can trigger a local APIC error
  88. * if the vector is zero. Mask LVTERR first to prevent this.
  89. */
  90. if (maxlvt >= 3) {
  91. v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
  92. apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
  93. }
  94. /*
  95. * Careful: we have to set masks only first to deassert
  96. * any level-triggered sources.
  97. */
  98. v = apic_read(APIC_LVTT);
  99. apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
  100. v = apic_read(APIC_LVT0);
  101. apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
  102. v = apic_read(APIC_LVT1);
  103. apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
  104. if (maxlvt >= 4) {
  105. v = apic_read(APIC_LVTPC);
  106. apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
  107. }
  108. /*
  109. * Clean APIC state for other OSs:
  110. */
  111. apic_write(APIC_LVTT, APIC_LVT_MASKED);
  112. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  113. apic_write(APIC_LVT1, APIC_LVT_MASKED);
  114. if (maxlvt >= 3)
  115. apic_write(APIC_LVTERR, APIC_LVT_MASKED);
  116. if (maxlvt >= 4)
  117. apic_write(APIC_LVTPC, APIC_LVT_MASKED);
  118. v = GET_APIC_VERSION(apic_read(APIC_LVR));
  119. apic_write(APIC_ESR, 0);
  120. apic_read(APIC_ESR);
  121. }
  122. void __init connect_bsp_APIC(void)
  123. {
  124. if (pic_mode) {
  125. /*
  126. * Do not trust the local APIC being empty at bootup.
  127. */
  128. clear_local_APIC();
  129. /*
  130. * PIC mode, enable APIC mode in the IMCR, i.e.
  131. * connect BSP's local APIC to INT and NMI lines.
  132. */
  133. apic_printk(APIC_VERBOSE, "leaving PIC mode, enabling APIC mode.\n");
  134. outb(0x70, 0x22);
  135. outb(0x01, 0x23);
  136. }
  137. }
  138. void disconnect_bsp_APIC(int virt_wire_setup)
  139. {
  140. if (pic_mode) {
  141. /*
  142. * Put the board back into PIC mode (has an effect
  143. * only on certain older boards). Note that APIC
  144. * interrupts, including IPIs, won't work beyond
  145. * this point! The only exception are INIT IPIs.
  146. */
  147. apic_printk(APIC_QUIET, "disabling APIC mode, entering PIC mode.\n");
  148. outb(0x70, 0x22);
  149. outb(0x00, 0x23);
  150. }
  151. else {
  152. /* Go back to Virtual Wire compatibility mode */
  153. unsigned long value;
  154. /* For the spurious interrupt use vector F, and enable it */
  155. value = apic_read(APIC_SPIV);
  156. value &= ~APIC_VECTOR_MASK;
  157. value |= APIC_SPIV_APIC_ENABLED;
  158. value |= 0xf;
  159. apic_write(APIC_SPIV, value);
  160. if (!virt_wire_setup) {
  161. /* For LVT0 make it edge triggered, active high, external and enabled */
  162. value = apic_read(APIC_LVT0);
  163. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  164. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  165. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED );
  166. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  167. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
  168. apic_write(APIC_LVT0, value);
  169. }
  170. else {
  171. /* Disable LVT0 */
  172. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  173. }
  174. /* For LVT1 make it edge triggered, active high, nmi and enabled */
  175. value = apic_read(APIC_LVT1);
  176. value &= ~(
  177. APIC_MODE_MASK | APIC_SEND_PENDING |
  178. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  179. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  180. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  181. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
  182. apic_write(APIC_LVT1, value);
  183. }
  184. }
  185. void disable_local_APIC(void)
  186. {
  187. unsigned int value;
  188. clear_local_APIC();
  189. /*
  190. * Disable APIC (implies clearing of registers
  191. * for 82489DX!).
  192. */
  193. value = apic_read(APIC_SPIV);
  194. value &= ~APIC_SPIV_APIC_ENABLED;
  195. apic_write(APIC_SPIV, value);
  196. }
  197. /*
  198. * This is to verify that we're looking at a real local APIC.
  199. * Check these against your board if the CPUs aren't getting
  200. * started for no apparent reason.
  201. */
  202. int __init verify_local_APIC(void)
  203. {
  204. unsigned int reg0, reg1;
  205. /*
  206. * The version register is read-only in a real APIC.
  207. */
  208. reg0 = apic_read(APIC_LVR);
  209. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
  210. apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
  211. reg1 = apic_read(APIC_LVR);
  212. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
  213. /*
  214. * The two version reads above should print the same
  215. * numbers. If the second one is different, then we
  216. * poke at a non-APIC.
  217. */
  218. if (reg1 != reg0)
  219. return 0;
  220. /*
  221. * Check if the version looks reasonably.
  222. */
  223. reg1 = GET_APIC_VERSION(reg0);
  224. if (reg1 == 0x00 || reg1 == 0xff)
  225. return 0;
  226. reg1 = get_maxlvt();
  227. if (reg1 < 0x02 || reg1 == 0xff)
  228. return 0;
  229. /*
  230. * The ID register is read/write in a real APIC.
  231. */
  232. reg0 = apic_read(APIC_ID);
  233. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
  234. apic_write(APIC_ID, reg0 ^ APIC_ID_MASK);
  235. reg1 = apic_read(APIC_ID);
  236. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
  237. apic_write(APIC_ID, reg0);
  238. if (reg1 != (reg0 ^ APIC_ID_MASK))
  239. return 0;
  240. /*
  241. * The next two are just to see if we have sane values.
  242. * They're only really relevant if we're in Virtual Wire
  243. * compatibility mode, but most boxes are anymore.
  244. */
  245. reg0 = apic_read(APIC_LVT0);
  246. apic_printk(APIC_DEBUG,"Getting LVT0: %x\n", reg0);
  247. reg1 = apic_read(APIC_LVT1);
  248. apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
  249. return 1;
  250. }
  251. void __init sync_Arb_IDs(void)
  252. {
  253. /* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 */
  254. unsigned int ver = GET_APIC_VERSION(apic_read(APIC_LVR));
  255. if (ver >= 0x14) /* P4 or higher */
  256. return;
  257. /*
  258. * Wait for idle.
  259. */
  260. apic_wait_icr_idle();
  261. apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
  262. apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG
  263. | APIC_DM_INIT);
  264. }
  265. extern void __error_in_apic_c (void);
  266. /*
  267. * An initial setup of the virtual wire mode.
  268. */
  269. void __init init_bsp_APIC(void)
  270. {
  271. unsigned int value;
  272. /*
  273. * Don't do the setup now if we have a SMP BIOS as the
  274. * through-I/O-APIC virtual wire mode might be active.
  275. */
  276. if (smp_found_config || !cpu_has_apic)
  277. return;
  278. value = apic_read(APIC_LVR);
  279. /*
  280. * Do not trust the local APIC being empty at bootup.
  281. */
  282. clear_local_APIC();
  283. /*
  284. * Enable APIC.
  285. */
  286. value = apic_read(APIC_SPIV);
  287. value &= ~APIC_VECTOR_MASK;
  288. value |= APIC_SPIV_APIC_ENABLED;
  289. value |= APIC_SPIV_FOCUS_DISABLED;
  290. value |= SPURIOUS_APIC_VECTOR;
  291. apic_write(APIC_SPIV, value);
  292. /*
  293. * Set up the virtual wire mode.
  294. */
  295. apic_write(APIC_LVT0, APIC_DM_EXTINT);
  296. value = APIC_DM_NMI;
  297. apic_write(APIC_LVT1, value);
  298. }
  299. void __cpuinit setup_local_APIC (void)
  300. {
  301. unsigned int value, maxlvt;
  302. int i, j;
  303. value = apic_read(APIC_LVR);
  304. if ((SPURIOUS_APIC_VECTOR & 0x0f) != 0x0f)
  305. __error_in_apic_c();
  306. /*
  307. * Double-check whether this APIC is really registered.
  308. * This is meaningless in clustered apic mode, so we skip it.
  309. */
  310. if (!apic_id_registered())
  311. BUG();
  312. /*
  313. * Intel recommends to set DFR, LDR and TPR before enabling
  314. * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
  315. * document number 292116). So here it goes...
  316. */
  317. init_apic_ldr();
  318. /*
  319. * Set Task Priority to 'accept all'. We never change this
  320. * later on.
  321. */
  322. value = apic_read(APIC_TASKPRI);
  323. value &= ~APIC_TPRI_MASK;
  324. apic_write(APIC_TASKPRI, value);
  325. /*
  326. * After a crash, we no longer service the interrupts and a pending
  327. * interrupt from previous kernel might still have ISR bit set.
  328. *
  329. * Most probably by now CPU has serviced that pending interrupt and
  330. * it might not have done the ack_APIC_irq() because it thought,
  331. * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
  332. * does not clear the ISR bit and cpu thinks it has already serivced
  333. * the interrupt. Hence a vector might get locked. It was noticed
  334. * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
  335. */
  336. for (i = APIC_ISR_NR - 1; i >= 0; i--) {
  337. value = apic_read(APIC_ISR + i*0x10);
  338. for (j = 31; j >= 0; j--) {
  339. if (value & (1<<j))
  340. ack_APIC_irq();
  341. }
  342. }
  343. /*
  344. * Now that we are all set up, enable the APIC
  345. */
  346. value = apic_read(APIC_SPIV);
  347. value &= ~APIC_VECTOR_MASK;
  348. /*
  349. * Enable APIC
  350. */
  351. value |= APIC_SPIV_APIC_ENABLED;
  352. /*
  353. * Some unknown Intel IO/APIC (or APIC) errata is biting us with
  354. * certain networking cards. If high frequency interrupts are
  355. * happening on a particular IOAPIC pin, plus the IOAPIC routing
  356. * entry is masked/unmasked at a high rate as well then sooner or
  357. * later IOAPIC line gets 'stuck', no more interrupts are received
  358. * from the device. If focus CPU is disabled then the hang goes
  359. * away, oh well :-(
  360. *
  361. * [ This bug can be reproduced easily with a level-triggered
  362. * PCI Ne2000 networking cards and PII/PIII processors, dual
  363. * BX chipset. ]
  364. */
  365. /*
  366. * Actually disabling the focus CPU check just makes the hang less
  367. * frequent as it makes the interrupt distributon model be more
  368. * like LRU than MRU (the short-term load is more even across CPUs).
  369. * See also the comment in end_level_ioapic_irq(). --macro
  370. */
  371. #if 1
  372. /* Enable focus processor (bit==0) */
  373. value &= ~APIC_SPIV_FOCUS_DISABLED;
  374. #else
  375. /* Disable focus processor (bit==1) */
  376. value |= APIC_SPIV_FOCUS_DISABLED;
  377. #endif
  378. /*
  379. * Set spurious IRQ vector
  380. */
  381. value |= SPURIOUS_APIC_VECTOR;
  382. apic_write(APIC_SPIV, value);
  383. /*
  384. * Set up LVT0, LVT1:
  385. *
  386. * set up through-local-APIC on the BP's LINT0. This is not
  387. * strictly necessary in pure symmetric-IO mode, but sometimes
  388. * we delegate interrupts to the 8259A.
  389. */
  390. /*
  391. * TODO: set up through-local-APIC from through-I/O-APIC? --macro
  392. */
  393. value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
  394. if (!smp_processor_id() && (pic_mode || !value)) {
  395. value = APIC_DM_EXTINT;
  396. apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", smp_processor_id());
  397. } else {
  398. value = APIC_DM_EXTINT | APIC_LVT_MASKED;
  399. apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", smp_processor_id());
  400. }
  401. apic_write(APIC_LVT0, value);
  402. /*
  403. * only the BP should see the LINT1 NMI signal, obviously.
  404. */
  405. if (!smp_processor_id())
  406. value = APIC_DM_NMI;
  407. else
  408. value = APIC_DM_NMI | APIC_LVT_MASKED;
  409. apic_write(APIC_LVT1, value);
  410. {
  411. unsigned oldvalue;
  412. maxlvt = get_maxlvt();
  413. oldvalue = apic_read(APIC_ESR);
  414. value = ERROR_APIC_VECTOR; // enables sending errors
  415. apic_write(APIC_LVTERR, value);
  416. /*
  417. * spec says clear errors after enabling vector.
  418. */
  419. if (maxlvt > 3)
  420. apic_write(APIC_ESR, 0);
  421. value = apic_read(APIC_ESR);
  422. if (value != oldvalue)
  423. apic_printk(APIC_VERBOSE,
  424. "ESR value after enabling vector: %08x, after %08x\n",
  425. oldvalue, value);
  426. }
  427. nmi_watchdog_default();
  428. if (nmi_watchdog == NMI_LOCAL_APIC)
  429. setup_apic_nmi_watchdog();
  430. apic_pm_activate();
  431. }
  432. #ifdef CONFIG_PM
  433. static struct {
  434. /* 'active' is true if the local APIC was enabled by us and
  435. not the BIOS; this signifies that we are also responsible
  436. for disabling it before entering apm/acpi suspend */
  437. int active;
  438. /* r/w apic fields */
  439. unsigned int apic_id;
  440. unsigned int apic_taskpri;
  441. unsigned int apic_ldr;
  442. unsigned int apic_dfr;
  443. unsigned int apic_spiv;
  444. unsigned int apic_lvtt;
  445. unsigned int apic_lvtpc;
  446. unsigned int apic_lvt0;
  447. unsigned int apic_lvt1;
  448. unsigned int apic_lvterr;
  449. unsigned int apic_tmict;
  450. unsigned int apic_tdcr;
  451. unsigned int apic_thmr;
  452. } apic_pm_state;
  453. static int lapic_suspend(struct sys_device *dev, pm_message_t state)
  454. {
  455. unsigned long flags;
  456. if (!apic_pm_state.active)
  457. return 0;
  458. apic_pm_state.apic_id = apic_read(APIC_ID);
  459. apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
  460. apic_pm_state.apic_ldr = apic_read(APIC_LDR);
  461. apic_pm_state.apic_dfr = apic_read(APIC_DFR);
  462. apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
  463. apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
  464. apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
  465. apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
  466. apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
  467. apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
  468. apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
  469. apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
  470. apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
  471. local_save_flags(flags);
  472. local_irq_disable();
  473. disable_local_APIC();
  474. local_irq_restore(flags);
  475. return 0;
  476. }
  477. static int lapic_resume(struct sys_device *dev)
  478. {
  479. unsigned int l, h;
  480. unsigned long flags;
  481. if (!apic_pm_state.active)
  482. return 0;
  483. local_irq_save(flags);
  484. rdmsr(MSR_IA32_APICBASE, l, h);
  485. l &= ~MSR_IA32_APICBASE_BASE;
  486. l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
  487. wrmsr(MSR_IA32_APICBASE, l, h);
  488. apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
  489. apic_write(APIC_ID, apic_pm_state.apic_id);
  490. apic_write(APIC_DFR, apic_pm_state.apic_dfr);
  491. apic_write(APIC_LDR, apic_pm_state.apic_ldr);
  492. apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
  493. apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
  494. apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
  495. apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
  496. apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
  497. apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
  498. apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
  499. apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
  500. apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
  501. apic_write(APIC_ESR, 0);
  502. apic_read(APIC_ESR);
  503. apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
  504. apic_write(APIC_ESR, 0);
  505. apic_read(APIC_ESR);
  506. local_irq_restore(flags);
  507. return 0;
  508. }
  509. static struct sysdev_class lapic_sysclass = {
  510. set_kset_name("lapic"),
  511. .resume = lapic_resume,
  512. .suspend = lapic_suspend,
  513. };
  514. static struct sys_device device_lapic = {
  515. .id = 0,
  516. .cls = &lapic_sysclass,
  517. };
  518. static void __cpuinit apic_pm_activate(void)
  519. {
  520. apic_pm_state.active = 1;
  521. }
  522. static int __init init_lapic_sysfs(void)
  523. {
  524. int error;
  525. if (!cpu_has_apic)
  526. return 0;
  527. /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
  528. error = sysdev_class_register(&lapic_sysclass);
  529. if (!error)
  530. error = sysdev_register(&device_lapic);
  531. return error;
  532. }
  533. device_initcall(init_lapic_sysfs);
  534. #else /* CONFIG_PM */
  535. static void apic_pm_activate(void) { }
  536. #endif /* CONFIG_PM */
  537. static int __init apic_set_verbosity(char *str)
  538. {
  539. if (strcmp("debug", str) == 0)
  540. apic_verbosity = APIC_DEBUG;
  541. else if (strcmp("verbose", str) == 0)
  542. apic_verbosity = APIC_VERBOSE;
  543. else
  544. printk(KERN_WARNING "APIC Verbosity level %s not recognised"
  545. " use apic=verbose or apic=debug", str);
  546. return 1;
  547. }
  548. __setup("apic=", apic_set_verbosity);
  549. /*
  550. * Detect and enable local APICs on non-SMP boards.
  551. * Original code written by Keir Fraser.
  552. * On AMD64 we trust the BIOS - if it says no APIC it is likely
  553. * not correctly set up (usually the APIC timer won't work etc.)
  554. */
  555. static int __init detect_init_APIC (void)
  556. {
  557. if (!cpu_has_apic) {
  558. printk(KERN_INFO "No local APIC present\n");
  559. return -1;
  560. }
  561. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  562. boot_cpu_id = 0;
  563. return 0;
  564. }
  565. void __init init_apic_mappings(void)
  566. {
  567. unsigned long apic_phys;
  568. /*
  569. * If no local APIC can be found then set up a fake all
  570. * zeroes page to simulate the local APIC and another
  571. * one for the IO-APIC.
  572. */
  573. if (!smp_found_config && detect_init_APIC()) {
  574. apic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE);
  575. apic_phys = __pa(apic_phys);
  576. } else
  577. apic_phys = mp_lapic_addr;
  578. set_fixmap_nocache(FIX_APIC_BASE, apic_phys);
  579. apic_printk(APIC_VERBOSE,"mapped APIC to %16lx (%16lx)\n", APIC_BASE, apic_phys);
  580. /*
  581. * Fetch the APIC ID of the BSP in case we have a
  582. * default configuration (or the MP table is broken).
  583. */
  584. boot_cpu_id = GET_APIC_ID(apic_read(APIC_ID));
  585. #ifdef CONFIG_X86_IO_APIC
  586. {
  587. unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
  588. int i;
  589. for (i = 0; i < nr_ioapics; i++) {
  590. if (smp_found_config) {
  591. ioapic_phys = mp_ioapics[i].mpc_apicaddr;
  592. } else {
  593. ioapic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE);
  594. ioapic_phys = __pa(ioapic_phys);
  595. }
  596. set_fixmap_nocache(idx, ioapic_phys);
  597. apic_printk(APIC_VERBOSE,"mapped IOAPIC to %016lx (%016lx)\n",
  598. __fix_to_virt(idx), ioapic_phys);
  599. idx++;
  600. }
  601. }
  602. #endif
  603. }
  604. /*
  605. * This function sets up the local APIC timer, with a timeout of
  606. * 'clocks' APIC bus clock. During calibration we actually call
  607. * this function twice on the boot CPU, once with a bogus timeout
  608. * value, second time for real. The other (noncalibrating) CPUs
  609. * call this function only once, with the real, calibrated value.
  610. *
  611. * We do reads before writes even if unnecessary, to get around the
  612. * P5 APIC double write bug.
  613. */
  614. #define APIC_DIVISOR 16
  615. static void __setup_APIC_LVTT(unsigned int clocks)
  616. {
  617. unsigned int lvtt_value, tmp_value, ver;
  618. int cpu = smp_processor_id();
  619. ver = GET_APIC_VERSION(apic_read(APIC_LVR));
  620. lvtt_value = APIC_LVT_TIMER_PERIODIC | LOCAL_TIMER_VECTOR;
  621. if (cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask))
  622. lvtt_value |= APIC_LVT_MASKED;
  623. apic_write(APIC_LVTT, lvtt_value);
  624. /*
  625. * Divide PICLK by 16
  626. */
  627. tmp_value = apic_read(APIC_TDCR);
  628. apic_write(APIC_TDCR, (tmp_value
  629. & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE))
  630. | APIC_TDR_DIV_16);
  631. apic_write(APIC_TMICT, clocks/APIC_DIVISOR);
  632. }
  633. static void setup_APIC_timer(unsigned int clocks)
  634. {
  635. unsigned long flags;
  636. local_irq_save(flags);
  637. /* wait for irq slice */
  638. if (vxtime.hpet_address && hpet_use_timer) {
  639. int trigger = hpet_readl(HPET_T0_CMP);
  640. while (hpet_readl(HPET_COUNTER) >= trigger)
  641. /* do nothing */ ;
  642. while (hpet_readl(HPET_COUNTER) < trigger)
  643. /* do nothing */ ;
  644. } else {
  645. int c1, c2;
  646. outb_p(0x00, 0x43);
  647. c2 = inb_p(0x40);
  648. c2 |= inb_p(0x40) << 8;
  649. do {
  650. c1 = c2;
  651. outb_p(0x00, 0x43);
  652. c2 = inb_p(0x40);
  653. c2 |= inb_p(0x40) << 8;
  654. } while (c2 - c1 < 300);
  655. }
  656. __setup_APIC_LVTT(clocks);
  657. /* Turn off PIT interrupt if we use APIC timer as main timer.
  658. Only works with the PM timer right now
  659. TBD fix it for HPET too. */
  660. if (vxtime.mode == VXTIME_PMTMR &&
  661. smp_processor_id() == boot_cpu_id &&
  662. apic_runs_main_timer == 1 &&
  663. !cpu_isset(boot_cpu_id, timer_interrupt_broadcast_ipi_mask)) {
  664. stop_timer_interrupt();
  665. apic_runs_main_timer++;
  666. }
  667. local_irq_restore(flags);
  668. }
  669. /*
  670. * In this function we calibrate APIC bus clocks to the external
  671. * timer. Unfortunately we cannot use jiffies and the timer irq
  672. * to calibrate, since some later bootup code depends on getting
  673. * the first irq? Ugh.
  674. *
  675. * We want to do the calibration only once since we
  676. * want to have local timer irqs syncron. CPUs connected
  677. * by the same APIC bus have the very same bus frequency.
  678. * And we want to have irqs off anyways, no accidental
  679. * APIC irq that way.
  680. */
  681. #define TICK_COUNT 100000000
  682. static int __init calibrate_APIC_clock(void)
  683. {
  684. int apic, apic_start, tsc, tsc_start;
  685. int result;
  686. /*
  687. * Put whatever arbitrary (but long enough) timeout
  688. * value into the APIC clock, we just want to get the
  689. * counter running for calibration.
  690. */
  691. __setup_APIC_LVTT(1000000000);
  692. apic_start = apic_read(APIC_TMCCT);
  693. #ifdef CONFIG_X86_PM_TIMER
  694. if (apic_calibrate_pmtmr && pmtmr_ioport) {
  695. pmtimer_wait(5000); /* 5ms wait */
  696. apic = apic_read(APIC_TMCCT);
  697. result = (apic_start - apic) * 1000L / 5;
  698. } else
  699. #endif
  700. {
  701. rdtscl(tsc_start);
  702. do {
  703. apic = apic_read(APIC_TMCCT);
  704. rdtscl(tsc);
  705. } while ((tsc - tsc_start) < TICK_COUNT &&
  706. (apic - apic_start) < TICK_COUNT);
  707. result = (apic_start - apic) * 1000L * cpu_khz /
  708. (tsc - tsc_start);
  709. }
  710. printk("result %d\n", result);
  711. printk(KERN_INFO "Detected %d.%03d MHz APIC timer.\n",
  712. result / 1000 / 1000, result / 1000 % 1000);
  713. return result * APIC_DIVISOR / HZ;
  714. }
  715. static unsigned int calibration_result;
  716. void __init setup_boot_APIC_clock (void)
  717. {
  718. if (disable_apic_timer) {
  719. printk(KERN_INFO "Disabling APIC timer\n");
  720. return;
  721. }
  722. printk(KERN_INFO "Using local APIC timer interrupts.\n");
  723. using_apic_timer = 1;
  724. local_irq_disable();
  725. calibration_result = calibrate_APIC_clock();
  726. /*
  727. * Now set up the timer for real.
  728. */
  729. setup_APIC_timer(calibration_result);
  730. local_irq_enable();
  731. }
  732. void __cpuinit setup_secondary_APIC_clock(void)
  733. {
  734. local_irq_disable(); /* FIXME: Do we need this? --RR */
  735. setup_APIC_timer(calibration_result);
  736. local_irq_enable();
  737. }
  738. void disable_APIC_timer(void)
  739. {
  740. if (using_apic_timer) {
  741. unsigned long v;
  742. v = apic_read(APIC_LVTT);
  743. /*
  744. * When an illegal vector value (0-15) is written to an LVT
  745. * entry and delivery mode is Fixed, the APIC may signal an
  746. * illegal vector error, with out regard to whether the mask
  747. * bit is set or whether an interrupt is actually seen on input.
  748. *
  749. * Boot sequence might call this function when the LVTT has
  750. * '0' vector value. So make sure vector field is set to
  751. * valid value.
  752. */
  753. v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  754. apic_write(APIC_LVTT, v);
  755. }
  756. }
  757. void enable_APIC_timer(void)
  758. {
  759. int cpu = smp_processor_id();
  760. if (using_apic_timer &&
  761. !cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) {
  762. unsigned long v;
  763. v = apic_read(APIC_LVTT);
  764. apic_write(APIC_LVTT, v & ~APIC_LVT_MASKED);
  765. }
  766. }
  767. void switch_APIC_timer_to_ipi(void *cpumask)
  768. {
  769. cpumask_t mask = *(cpumask_t *)cpumask;
  770. int cpu = smp_processor_id();
  771. if (cpu_isset(cpu, mask) &&
  772. !cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) {
  773. disable_APIC_timer();
  774. cpu_set(cpu, timer_interrupt_broadcast_ipi_mask);
  775. }
  776. }
  777. EXPORT_SYMBOL(switch_APIC_timer_to_ipi);
  778. void smp_send_timer_broadcast_ipi(void)
  779. {
  780. cpumask_t mask;
  781. cpus_and(mask, cpu_online_map, timer_interrupt_broadcast_ipi_mask);
  782. if (!cpus_empty(mask)) {
  783. send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
  784. }
  785. }
  786. void switch_ipi_to_APIC_timer(void *cpumask)
  787. {
  788. cpumask_t mask = *(cpumask_t *)cpumask;
  789. int cpu = smp_processor_id();
  790. if (cpu_isset(cpu, mask) &&
  791. cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) {
  792. cpu_clear(cpu, timer_interrupt_broadcast_ipi_mask);
  793. enable_APIC_timer();
  794. }
  795. }
  796. EXPORT_SYMBOL(switch_ipi_to_APIC_timer);
  797. int setup_profiling_timer(unsigned int multiplier)
  798. {
  799. return -EINVAL;
  800. }
  801. void setup_APIC_extened_lvt(unsigned char lvt_off, unsigned char vector,
  802. unsigned char msg_type, unsigned char mask)
  803. {
  804. unsigned long reg = (lvt_off << 4) + K8_APIC_EXT_LVT_BASE;
  805. unsigned int v = (mask << 16) | (msg_type << 8) | vector;
  806. apic_write(reg, v);
  807. }
  808. #undef APIC_DIVISOR
  809. /*
  810. * Local timer interrupt handler. It does both profiling and
  811. * process statistics/rescheduling.
  812. *
  813. * We do profiling in every local tick, statistics/rescheduling
  814. * happen only every 'profiling multiplier' ticks. The default
  815. * multiplier is 1 and it can be changed by writing the new multiplier
  816. * value into /proc/profile.
  817. */
  818. void smp_local_timer_interrupt(struct pt_regs *regs)
  819. {
  820. profile_tick(CPU_PROFILING, regs);
  821. #ifdef CONFIG_SMP
  822. update_process_times(user_mode(regs));
  823. #endif
  824. if (apic_runs_main_timer > 1 && smp_processor_id() == boot_cpu_id)
  825. main_timer_handler(regs);
  826. /*
  827. * We take the 'long' return path, and there every subsystem
  828. * grabs the appropriate locks (kernel lock/ irq lock).
  829. *
  830. * we might want to decouple profiling from the 'long path',
  831. * and do the profiling totally in assembly.
  832. *
  833. * Currently this isn't too much of an issue (performance wise),
  834. * we can take more than 100K local irqs per second on a 100 MHz P5.
  835. */
  836. }
  837. /*
  838. * Local APIC timer interrupt. This is the most natural way for doing
  839. * local interrupts, but local timer interrupts can be emulated by
  840. * broadcast interrupts too. [in case the hw doesn't support APIC timers]
  841. *
  842. * [ if a single-CPU system runs an SMP kernel then we call the local
  843. * interrupt as well. Thus we cannot inline the local irq ... ]
  844. */
  845. void smp_apic_timer_interrupt(struct pt_regs *regs)
  846. {
  847. /*
  848. * the NMI deadlock-detector uses this.
  849. */
  850. add_pda(apic_timer_irqs, 1);
  851. /*
  852. * NOTE! We'd better ACK the irq immediately,
  853. * because timer handling can be slow.
  854. */
  855. ack_APIC_irq();
  856. /*
  857. * update_process_times() expects us to have done irq_enter().
  858. * Besides, if we don't timer interrupts ignore the global
  859. * interrupt lock, which is the WrongThing (tm) to do.
  860. */
  861. exit_idle();
  862. irq_enter();
  863. smp_local_timer_interrupt(regs);
  864. irq_exit();
  865. }
  866. /*
  867. * apic_is_clustered_box() -- Check if we can expect good TSC
  868. *
  869. * Thus far, the major user of this is IBM's Summit2 series:
  870. *
  871. * Clustered boxes may have unsynced TSC problems if they are
  872. * multi-chassis. Use available data to take a good guess.
  873. * If in doubt, go HPET.
  874. */
  875. __cpuinit int apic_is_clustered_box(void)
  876. {
  877. int i, clusters, zeros;
  878. unsigned id;
  879. DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);
  880. bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
  881. for (i = 0; i < NR_CPUS; i++) {
  882. id = bios_cpu_apicid[i];
  883. if (id != BAD_APICID)
  884. __set_bit(APIC_CLUSTERID(id), clustermap);
  885. }
  886. /* Problem: Partially populated chassis may not have CPUs in some of
  887. * the APIC clusters they have been allocated. Only present CPUs have
  888. * bios_cpu_apicid entries, thus causing zeroes in the bitmap. Since
  889. * clusters are allocated sequentially, count zeros only if they are
  890. * bounded by ones.
  891. */
  892. clusters = 0;
  893. zeros = 0;
  894. for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
  895. if (test_bit(i, clustermap)) {
  896. clusters += 1 + zeros;
  897. zeros = 0;
  898. } else
  899. ++zeros;
  900. }
  901. /*
  902. * If clusters > 2, then should be multi-chassis.
  903. * May have to revisit this when multi-core + hyperthreaded CPUs come
  904. * out, but AFAIK this will work even for them.
  905. */
  906. return (clusters > 2);
  907. }
  908. /*
  909. * This interrupt should _never_ happen with our APIC/SMP architecture
  910. */
  911. asmlinkage void smp_spurious_interrupt(void)
  912. {
  913. unsigned int v;
  914. exit_idle();
  915. irq_enter();
  916. /*
  917. * Check if this really is a spurious interrupt and ACK it
  918. * if it is a vectored one. Just in case...
  919. * Spurious interrupts should not be ACKed.
  920. */
  921. v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
  922. if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
  923. ack_APIC_irq();
  924. #if 0
  925. static unsigned long last_warning;
  926. static unsigned long skipped;
  927. /* see sw-dev-man vol 3, chapter 7.4.13.5 */
  928. if (time_before(last_warning+30*HZ,jiffies)) {
  929. printk(KERN_INFO "spurious APIC interrupt on CPU#%d, %ld skipped.\n",
  930. smp_processor_id(), skipped);
  931. last_warning = jiffies;
  932. skipped = 0;
  933. } else {
  934. skipped++;
  935. }
  936. #endif
  937. irq_exit();
  938. }
  939. /*
  940. * This interrupt should never happen with our APIC/SMP architecture
  941. */
  942. asmlinkage void smp_error_interrupt(void)
  943. {
  944. unsigned int v, v1;
  945. exit_idle();
  946. irq_enter();
  947. /* First tickle the hardware, only then report what went on. -- REW */
  948. v = apic_read(APIC_ESR);
  949. apic_write(APIC_ESR, 0);
  950. v1 = apic_read(APIC_ESR);
  951. ack_APIC_irq();
  952. atomic_inc(&irq_err_count);
  953. /* Here is what the APIC error bits mean:
  954. 0: Send CS error
  955. 1: Receive CS error
  956. 2: Send accept error
  957. 3: Receive accept error
  958. 4: Reserved
  959. 5: Send illegal vector
  960. 6: Received illegal vector
  961. 7: Illegal register address
  962. */
  963. printk (KERN_DEBUG "APIC error on CPU%d: %02x(%02x)\n",
  964. smp_processor_id(), v , v1);
  965. irq_exit();
  966. }
  967. int disable_apic;
  968. /*
  969. * This initializes the IO-APIC and APIC hardware if this is
  970. * a UP kernel.
  971. */
  972. int __init APIC_init_uniprocessor (void)
  973. {
  974. if (disable_apic) {
  975. printk(KERN_INFO "Apic disabled\n");
  976. return -1;
  977. }
  978. if (!cpu_has_apic) {
  979. disable_apic = 1;
  980. printk(KERN_INFO "Apic disabled by BIOS\n");
  981. return -1;
  982. }
  983. verify_local_APIC();
  984. connect_bsp_APIC();
  985. phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id);
  986. apic_write(APIC_ID, SET_APIC_ID(boot_cpu_id));
  987. setup_local_APIC();
  988. #ifdef CONFIG_X86_IO_APIC
  989. if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
  990. setup_IO_APIC();
  991. else
  992. nr_ioapics = 0;
  993. #endif
  994. setup_boot_APIC_clock();
  995. check_nmi_watchdog();
  996. return 0;
  997. }
  998. static __init int setup_disableapic(char *str)
  999. {
  1000. disable_apic = 1;
  1001. return 1;
  1002. }
  1003. static __init int setup_nolapic(char *str)
  1004. {
  1005. disable_apic = 1;
  1006. return 1;
  1007. }
  1008. static __init int setup_noapictimer(char *str)
  1009. {
  1010. if (str[0] != ' ' && str[0] != 0)
  1011. return 0;
  1012. disable_apic_timer = 1;
  1013. return 1;
  1014. }
  1015. static __init int setup_apicmaintimer(char *str)
  1016. {
  1017. apic_runs_main_timer = 1;
  1018. nohpet = 1;
  1019. return 1;
  1020. }
  1021. __setup("apicmaintimer", setup_apicmaintimer);
  1022. static __init int setup_noapicmaintimer(char *str)
  1023. {
  1024. apic_runs_main_timer = -1;
  1025. return 1;
  1026. }
  1027. __setup("noapicmaintimer", setup_noapicmaintimer);
  1028. static __init int setup_apicpmtimer(char *s)
  1029. {
  1030. apic_calibrate_pmtmr = 1;
  1031. notsc_setup(NULL);
  1032. return setup_apicmaintimer(NULL);
  1033. }
  1034. __setup("apicpmtimer", setup_apicpmtimer);
  1035. /* dummy parsing: see setup.c */
  1036. __setup("disableapic", setup_disableapic);
  1037. __setup("nolapic", setup_nolapic); /* same as disableapic, for compatibility */
  1038. __setup("noapictimer", setup_noapictimer);
  1039. /* no "lapic" flag - we only use the lapic when the BIOS tells us so. */