apic.c 58 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409
  1. /*
  2. * Local APIC handling, local APIC timers
  3. *
  4. * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
  5. *
  6. * Fixes
  7. * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
  8. * thanks to Eric Gilmore
  9. * and Rolf G. Tews
  10. * for testing these extensively.
  11. * Maciej W. Rozycki : Various updates and fixes.
  12. * Mikael Pettersson : Power Management for UP-APIC.
  13. * Pavel Machek and
  14. * Mikael Pettersson : PM converted to driver model.
  15. */
  16. #include <linux/perf_event.h>
  17. #include <linux/kernel_stat.h>
  18. #include <linux/mc146818rtc.h>
  19. #include <linux/acpi_pmtmr.h>
  20. #include <linux/clockchips.h>
  21. #include <linux/interrupt.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/ftrace.h>
  24. #include <linux/ioport.h>
  25. #include <linux/module.h>
  26. #include <linux/syscore_ops.h>
  27. #include <linux/delay.h>
  28. #include <linux/timex.h>
  29. #include <linux/i8253.h>
  30. #include <linux/dmar.h>
  31. #include <linux/init.h>
  32. #include <linux/cpu.h>
  33. #include <linux/dmi.h>
  34. #include <linux/smp.h>
  35. #include <linux/mm.h>
  36. #include <asm/perf_event.h>
  37. #include <asm/x86_init.h>
  38. #include <asm/pgalloc.h>
  39. #include <linux/atomic.h>
  40. #include <asm/mpspec.h>
  41. #include <asm/i8259.h>
  42. #include <asm/proto.h>
  43. #include <asm/apic.h>
  44. #include <asm/io_apic.h>
  45. #include <asm/desc.h>
  46. #include <asm/hpet.h>
  47. #include <asm/idle.h>
  48. #include <asm/mtrr.h>
  49. #include <asm/time.h>
  50. #include <asm/smp.h>
  51. #include <asm/mce.h>
  52. #include <asm/tsc.h>
  53. #include <asm/hypervisor.h>
  54. unsigned int num_processors;
  55. unsigned disabled_cpus __cpuinitdata;
  56. /* Processor that is doing the boot up */
  57. unsigned int boot_cpu_physical_apicid = -1U;
  58. /*
  59. * The highest APIC ID seen during enumeration.
  60. */
  61. unsigned int max_physical_apicid;
  62. /*
  63. * Bitmask of physically existing CPUs:
  64. */
  65. physid_mask_t phys_cpu_present_map;
  66. /*
  67. * Map cpu index to physical APIC ID
  68. */
  69. DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
  70. DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
  71. EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
  72. EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  73. #ifdef CONFIG_X86_32
  74. /*
  75. * On x86_32, the mapping between cpu and logical apicid may vary
  76. * depending on apic in use. The following early percpu variable is
  77. * used for the mapping. This is where the behaviors of x86_64 and 32
  78. * actually diverge. Let's keep it ugly for now.
  79. */
  80. DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
  81. /*
  82. * Knob to control our willingness to enable the local APIC.
  83. *
  84. * +1=force-enable
  85. */
  86. static int force_enable_local_apic __initdata;
  87. /*
  88. * APIC command line parameters
  89. */
  90. static int __init parse_lapic(char *arg)
  91. {
  92. force_enable_local_apic = 1;
  93. return 0;
  94. }
  95. early_param("lapic", parse_lapic);
  96. /* Local APIC was disabled by the BIOS and enabled by the kernel */
  97. static int enabled_via_apicbase;
  98. /*
  99. * Handle interrupt mode configuration register (IMCR).
  100. * This register controls whether the interrupt signals
  101. * that reach the BSP come from the master PIC or from the
  102. * local APIC. Before entering Symmetric I/O Mode, either
  103. * the BIOS or the operating system must switch out of
  104. * PIC Mode by changing the IMCR.
  105. */
  106. static inline void imcr_pic_to_apic(void)
  107. {
  108. /* select IMCR register */
  109. outb(0x70, 0x22);
  110. /* NMI and 8259 INTR go through APIC */
  111. outb(0x01, 0x23);
  112. }
  113. static inline void imcr_apic_to_pic(void)
  114. {
  115. /* select IMCR register */
  116. outb(0x70, 0x22);
  117. /* NMI and 8259 INTR go directly to BSP */
  118. outb(0x00, 0x23);
  119. }
  120. #endif
  121. #ifdef CONFIG_X86_64
  122. static int apic_calibrate_pmtmr __initdata;
  123. static __init int setup_apicpmtimer(char *s)
  124. {
  125. apic_calibrate_pmtmr = 1;
  126. notsc_setup(NULL);
  127. return 0;
  128. }
  129. __setup("apicpmtimer", setup_apicpmtimer);
  130. #endif
  131. int x2apic_mode;
  132. #ifdef CONFIG_X86_X2APIC
  133. /* x2apic enabled before OS handover */
  134. static int x2apic_preenabled;
  135. static __init int setup_nox2apic(char *str)
  136. {
  137. if (x2apic_enabled()) {
  138. pr_warning("Bios already enabled x2apic, "
  139. "can't enforce nox2apic");
  140. return 0;
  141. }
  142. setup_clear_cpu_cap(X86_FEATURE_X2APIC);
  143. return 0;
  144. }
  145. early_param("nox2apic", setup_nox2apic);
  146. #endif
  147. unsigned long mp_lapic_addr;
  148. int disable_apic;
  149. /* Disable local APIC timer from the kernel commandline or via dmi quirk */
  150. static int disable_apic_timer __initdata;
  151. /* Local APIC timer works in C2 */
  152. int local_apic_timer_c2_ok;
  153. EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
  154. int first_system_vector = 0xfe;
  155. /*
  156. * Debug level, exported for io_apic.c
  157. */
  158. unsigned int apic_verbosity;
  159. int pic_mode;
  160. /* Have we found an MP table */
  161. int smp_found_config;
  162. static struct resource lapic_resource = {
  163. .name = "Local APIC",
  164. .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
  165. };
  166. unsigned int lapic_timer_frequency = 0;
  167. static void apic_pm_activate(void);
  168. static unsigned long apic_phys;
  169. /*
  170. * Get the LAPIC version
  171. */
  172. static inline int lapic_get_version(void)
  173. {
  174. return GET_APIC_VERSION(apic_read(APIC_LVR));
  175. }
  176. /*
  177. * Check, if the APIC is integrated or a separate chip
  178. */
  179. static inline int lapic_is_integrated(void)
  180. {
  181. #ifdef CONFIG_X86_64
  182. return 1;
  183. #else
  184. return APIC_INTEGRATED(lapic_get_version());
  185. #endif
  186. }
  187. /*
  188. * Check, whether this is a modern or a first generation APIC
  189. */
  190. static int modern_apic(void)
  191. {
  192. /* AMD systems use old APIC versions, so check the CPU */
  193. if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
  194. boot_cpu_data.x86 >= 0xf)
  195. return 1;
  196. return lapic_get_version() >= 0x14;
  197. }
  198. /*
  199. * right after this call apic become NOOP driven
  200. * so apic->write/read doesn't do anything
  201. */
  202. static void __init apic_disable(void)
  203. {
  204. pr_info("APIC: switched to apic NOOP\n");
  205. apic = &apic_noop;
  206. }
  207. void native_apic_wait_icr_idle(void)
  208. {
  209. while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
  210. cpu_relax();
  211. }
  212. u32 native_safe_apic_wait_icr_idle(void)
  213. {
  214. u32 send_status;
  215. int timeout;
  216. timeout = 0;
  217. do {
  218. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  219. if (!send_status)
  220. break;
  221. udelay(100);
  222. } while (timeout++ < 1000);
  223. return send_status;
  224. }
  225. void native_apic_icr_write(u32 low, u32 id)
  226. {
  227. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
  228. apic_write(APIC_ICR, low);
  229. }
  230. u64 native_apic_icr_read(void)
  231. {
  232. u32 icr1, icr2;
  233. icr2 = apic_read(APIC_ICR2);
  234. icr1 = apic_read(APIC_ICR);
  235. return icr1 | ((u64)icr2 << 32);
  236. }
  237. #ifdef CONFIG_X86_32
  238. /**
  239. * get_physical_broadcast - Get number of physical broadcast IDs
  240. */
  241. int get_physical_broadcast(void)
  242. {
  243. return modern_apic() ? 0xff : 0xf;
  244. }
  245. #endif
  246. /**
  247. * lapic_get_maxlvt - get the maximum number of local vector table entries
  248. */
  249. int lapic_get_maxlvt(void)
  250. {
  251. unsigned int v;
  252. v = apic_read(APIC_LVR);
  253. /*
  254. * - we always have APIC integrated on 64bit mode
  255. * - 82489DXs do not report # of LVT entries
  256. */
  257. return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
  258. }
  259. /*
  260. * Local APIC timer
  261. */
  262. /* Clock divisor */
  263. #define APIC_DIVISOR 16
  264. /*
  265. * This function sets up the local APIC timer, with a timeout of
  266. * 'clocks' APIC bus clock. During calibration we actually call
  267. * this function twice on the boot CPU, once with a bogus timeout
  268. * value, second time for real. The other (noncalibrating) CPUs
  269. * call this function only once, with the real, calibrated value.
  270. *
  271. * We do reads before writes even if unnecessary, to get around the
  272. * P5 APIC double write bug.
  273. */
  274. static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
  275. {
  276. unsigned int lvtt_value, tmp_value;
  277. lvtt_value = LOCAL_TIMER_VECTOR;
  278. if (!oneshot)
  279. lvtt_value |= APIC_LVT_TIMER_PERIODIC;
  280. if (!lapic_is_integrated())
  281. lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
  282. if (!irqen)
  283. lvtt_value |= APIC_LVT_MASKED;
  284. apic_write(APIC_LVTT, lvtt_value);
  285. /*
  286. * Divide PICLK by 16
  287. */
  288. tmp_value = apic_read(APIC_TDCR);
  289. apic_write(APIC_TDCR,
  290. (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
  291. APIC_TDR_DIV_16);
  292. if (!oneshot)
  293. apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
  294. }
  295. /*
  296. * Setup extended LVT, AMD specific
  297. *
  298. * Software should use the LVT offsets the BIOS provides. The offsets
  299. * are determined by the subsystems using it like those for MCE
  300. * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
  301. * are supported. Beginning with family 10h at least 4 offsets are
  302. * available.
  303. *
  304. * Since the offsets must be consistent for all cores, we keep track
  305. * of the LVT offsets in software and reserve the offset for the same
  306. * vector also to be used on other cores. An offset is freed by
  307. * setting the entry to APIC_EILVT_MASKED.
  308. *
  309. * If the BIOS is right, there should be no conflicts. Otherwise a
  310. * "[Firmware Bug]: ..." error message is generated. However, if
  311. * software does not properly determines the offsets, it is not
  312. * necessarily a BIOS bug.
  313. */
  314. static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
  315. static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
  316. {
  317. return (old & APIC_EILVT_MASKED)
  318. || (new == APIC_EILVT_MASKED)
  319. || ((new & ~APIC_EILVT_MASKED) == old);
  320. }
  321. static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
  322. {
  323. unsigned int rsvd; /* 0: uninitialized */
  324. if (offset >= APIC_EILVT_NR_MAX)
  325. return ~0;
  326. rsvd = atomic_read(&eilvt_offsets[offset]) & ~APIC_EILVT_MASKED;
  327. do {
  328. if (rsvd &&
  329. !eilvt_entry_is_changeable(rsvd, new))
  330. /* may not change if vectors are different */
  331. return rsvd;
  332. rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
  333. } while (rsvd != new);
  334. return new;
  335. }
  336. /*
  337. * If mask=1, the LVT entry does not generate interrupts while mask=0
  338. * enables the vector. See also the BKDGs. Must be called with
  339. * preemption disabled.
  340. */
  341. int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
  342. {
  343. unsigned long reg = APIC_EILVTn(offset);
  344. unsigned int new, old, reserved;
  345. new = (mask << 16) | (msg_type << 8) | vector;
  346. old = apic_read(reg);
  347. reserved = reserve_eilvt_offset(offset, new);
  348. if (reserved != new) {
  349. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  350. "vector 0x%x, but the register is already in use for "
  351. "vector 0x%x on another cpu\n",
  352. smp_processor_id(), reg, offset, new, reserved);
  353. return -EINVAL;
  354. }
  355. if (!eilvt_entry_is_changeable(old, new)) {
  356. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  357. "vector 0x%x, but the register is already in use for "
  358. "vector 0x%x on this cpu\n",
  359. smp_processor_id(), reg, offset, new, old);
  360. return -EBUSY;
  361. }
  362. apic_write(reg, new);
  363. return 0;
  364. }
  365. EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
  366. /*
  367. * Program the next event, relative to now
  368. */
  369. static int lapic_next_event(unsigned long delta,
  370. struct clock_event_device *evt)
  371. {
  372. apic_write(APIC_TMICT, delta);
  373. return 0;
  374. }
  375. /*
  376. * Setup the lapic timer in periodic or oneshot mode
  377. */
  378. static void lapic_timer_setup(enum clock_event_mode mode,
  379. struct clock_event_device *evt)
  380. {
  381. unsigned long flags;
  382. unsigned int v;
  383. /* Lapic used as dummy for broadcast ? */
  384. if (evt->features & CLOCK_EVT_FEAT_DUMMY)
  385. return;
  386. local_irq_save(flags);
  387. switch (mode) {
  388. case CLOCK_EVT_MODE_PERIODIC:
  389. case CLOCK_EVT_MODE_ONESHOT:
  390. __setup_APIC_LVTT(lapic_timer_frequency,
  391. mode != CLOCK_EVT_MODE_PERIODIC, 1);
  392. break;
  393. case CLOCK_EVT_MODE_UNUSED:
  394. case CLOCK_EVT_MODE_SHUTDOWN:
  395. v = apic_read(APIC_LVTT);
  396. v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  397. apic_write(APIC_LVTT, v);
  398. apic_write(APIC_TMICT, 0);
  399. break;
  400. case CLOCK_EVT_MODE_RESUME:
  401. /* Nothing to do here */
  402. break;
  403. }
  404. local_irq_restore(flags);
  405. }
  406. /*
  407. * Local APIC timer broadcast function
  408. */
  409. static void lapic_timer_broadcast(const struct cpumask *mask)
  410. {
  411. #ifdef CONFIG_SMP
  412. apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
  413. #endif
  414. }
  415. /*
  416. * The local apic timer can be used for any function which is CPU local.
  417. */
  418. static struct clock_event_device lapic_clockevent = {
  419. .name = "lapic",
  420. .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
  421. | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
  422. .shift = 32,
  423. .set_mode = lapic_timer_setup,
  424. .set_next_event = lapic_next_event,
  425. .broadcast = lapic_timer_broadcast,
  426. .rating = 100,
  427. .irq = -1,
  428. };
  429. static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
  430. /*
  431. * Setup the local APIC timer for this CPU. Copy the initialized values
  432. * of the boot CPU and register the clock event in the framework.
  433. */
  434. static void __cpuinit setup_APIC_timer(void)
  435. {
  436. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  437. if (this_cpu_has(X86_FEATURE_ARAT)) {
  438. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
  439. /* Make LAPIC timer preferrable over percpu HPET */
  440. lapic_clockevent.rating = 150;
  441. }
  442. memcpy(levt, &lapic_clockevent, sizeof(*levt));
  443. levt->cpumask = cpumask_of(smp_processor_id());
  444. clockevents_register_device(levt);
  445. }
  446. /*
  447. * In this functions we calibrate APIC bus clocks to the external timer.
  448. *
  449. * We want to do the calibration only once since we want to have local timer
  450. * irqs syncron. CPUs connected by the same APIC bus have the very same bus
  451. * frequency.
  452. *
  453. * This was previously done by reading the PIT/HPET and waiting for a wrap
  454. * around to find out, that a tick has elapsed. I have a box, where the PIT
  455. * readout is broken, so it never gets out of the wait loop again. This was
  456. * also reported by others.
  457. *
  458. * Monitoring the jiffies value is inaccurate and the clockevents
  459. * infrastructure allows us to do a simple substitution of the interrupt
  460. * handler.
  461. *
  462. * The calibration routine also uses the pm_timer when possible, as the PIT
  463. * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
  464. * back to normal later in the boot process).
  465. */
  466. #define LAPIC_CAL_LOOPS (HZ/10)
  467. static __initdata int lapic_cal_loops = -1;
  468. static __initdata long lapic_cal_t1, lapic_cal_t2;
  469. static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
  470. static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
  471. static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
  472. /*
  473. * Temporary interrupt handler.
  474. */
  475. static void __init lapic_cal_handler(struct clock_event_device *dev)
  476. {
  477. unsigned long long tsc = 0;
  478. long tapic = apic_read(APIC_TMCCT);
  479. unsigned long pm = acpi_pm_read_early();
  480. if (cpu_has_tsc)
  481. rdtscll(tsc);
  482. switch (lapic_cal_loops++) {
  483. case 0:
  484. lapic_cal_t1 = tapic;
  485. lapic_cal_tsc1 = tsc;
  486. lapic_cal_pm1 = pm;
  487. lapic_cal_j1 = jiffies;
  488. break;
  489. case LAPIC_CAL_LOOPS:
  490. lapic_cal_t2 = tapic;
  491. lapic_cal_tsc2 = tsc;
  492. if (pm < lapic_cal_pm1)
  493. pm += ACPI_PM_OVRRUN;
  494. lapic_cal_pm2 = pm;
  495. lapic_cal_j2 = jiffies;
  496. break;
  497. }
  498. }
  499. static int __init
  500. calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
  501. {
  502. const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
  503. const long pm_thresh = pm_100ms / 100;
  504. unsigned long mult;
  505. u64 res;
  506. #ifndef CONFIG_X86_PM_TIMER
  507. return -1;
  508. #endif
  509. apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
  510. /* Check, if the PM timer is available */
  511. if (!deltapm)
  512. return -1;
  513. mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
  514. if (deltapm > (pm_100ms - pm_thresh) &&
  515. deltapm < (pm_100ms + pm_thresh)) {
  516. apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
  517. return 0;
  518. }
  519. res = (((u64)deltapm) * mult) >> 22;
  520. do_div(res, 1000000);
  521. pr_warning("APIC calibration not consistent "
  522. "with PM-Timer: %ldms instead of 100ms\n",(long)res);
  523. /* Correct the lapic counter value */
  524. res = (((u64)(*delta)) * pm_100ms);
  525. do_div(res, deltapm);
  526. pr_info("APIC delta adjusted to PM-Timer: "
  527. "%lu (%ld)\n", (unsigned long)res, *delta);
  528. *delta = (long)res;
  529. /* Correct the tsc counter value */
  530. if (cpu_has_tsc) {
  531. res = (((u64)(*deltatsc)) * pm_100ms);
  532. do_div(res, deltapm);
  533. apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
  534. "PM-Timer: %lu (%ld)\n",
  535. (unsigned long)res, *deltatsc);
  536. *deltatsc = (long)res;
  537. }
  538. return 0;
  539. }
  540. static int __init calibrate_APIC_clock(void)
  541. {
  542. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  543. void (*real_handler)(struct clock_event_device *dev);
  544. unsigned long deltaj;
  545. long delta, deltatsc;
  546. int pm_referenced = 0;
  547. /**
  548. * check if lapic timer has already been calibrated by platform
  549. * specific routine, such as tsc calibration code. if so, we just fill
  550. * in the clockevent structure and return.
  551. */
  552. if (lapic_timer_frequency) {
  553. apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
  554. lapic_timer_frequency);
  555. lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
  556. TICK_NSEC, lapic_clockevent.shift);
  557. lapic_clockevent.max_delta_ns =
  558. clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
  559. lapic_clockevent.min_delta_ns =
  560. clockevent_delta2ns(0xF, &lapic_clockevent);
  561. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
  562. return 0;
  563. }
  564. local_irq_disable();
  565. /* Replace the global interrupt handler */
  566. real_handler = global_clock_event->event_handler;
  567. global_clock_event->event_handler = lapic_cal_handler;
  568. /*
  569. * Setup the APIC counter to maximum. There is no way the lapic
  570. * can underflow in the 100ms detection time frame
  571. */
  572. __setup_APIC_LVTT(0xffffffff, 0, 0);
  573. /* Let the interrupts run */
  574. local_irq_enable();
  575. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  576. cpu_relax();
  577. local_irq_disable();
  578. /* Restore the real event handler */
  579. global_clock_event->event_handler = real_handler;
  580. /* Build delta t1-t2 as apic timer counts down */
  581. delta = lapic_cal_t1 - lapic_cal_t2;
  582. apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
  583. deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
  584. /* we trust the PM based calibration if possible */
  585. pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
  586. &delta, &deltatsc);
  587. /* Calculate the scaled math multiplication factor */
  588. lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
  589. lapic_clockevent.shift);
  590. lapic_clockevent.max_delta_ns =
  591. clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
  592. lapic_clockevent.min_delta_ns =
  593. clockevent_delta2ns(0xF, &lapic_clockevent);
  594. lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
  595. apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
  596. apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
  597. apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
  598. lapic_timer_frequency);
  599. if (cpu_has_tsc) {
  600. apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
  601. "%ld.%04ld MHz.\n",
  602. (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
  603. (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
  604. }
  605. apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
  606. "%u.%04u MHz.\n",
  607. lapic_timer_frequency / (1000000 / HZ),
  608. lapic_timer_frequency % (1000000 / HZ));
  609. /*
  610. * Do a sanity check on the APIC calibration result
  611. */
  612. if (lapic_timer_frequency < (1000000 / HZ)) {
  613. local_irq_enable();
  614. pr_warning("APIC frequency too slow, disabling apic timer\n");
  615. return -1;
  616. }
  617. levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
  618. /*
  619. * PM timer calibration failed or not turned on
  620. * so lets try APIC timer based calibration
  621. */
  622. if (!pm_referenced) {
  623. apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
  624. /*
  625. * Setup the apic timer manually
  626. */
  627. levt->event_handler = lapic_cal_handler;
  628. lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
  629. lapic_cal_loops = -1;
  630. /* Let the interrupts run */
  631. local_irq_enable();
  632. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  633. cpu_relax();
  634. /* Stop the lapic timer */
  635. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
  636. /* Jiffies delta */
  637. deltaj = lapic_cal_j2 - lapic_cal_j1;
  638. apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
  639. /* Check, if the jiffies result is consistent */
  640. if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
  641. apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
  642. else
  643. levt->features |= CLOCK_EVT_FEAT_DUMMY;
  644. } else
  645. local_irq_enable();
  646. if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
  647. pr_warning("APIC timer disabled due to verification failure\n");
  648. return -1;
  649. }
  650. return 0;
  651. }
  652. /*
  653. * Setup the boot APIC
  654. *
  655. * Calibrate and verify the result.
  656. */
  657. void __init setup_boot_APIC_clock(void)
  658. {
  659. /*
  660. * The local apic timer can be disabled via the kernel
  661. * commandline or from the CPU detection code. Register the lapic
  662. * timer as a dummy clock event source on SMP systems, so the
  663. * broadcast mechanism is used. On UP systems simply ignore it.
  664. */
  665. if (disable_apic_timer) {
  666. pr_info("Disabling APIC timer\n");
  667. /* No broadcast on UP ! */
  668. if (num_possible_cpus() > 1) {
  669. lapic_clockevent.mult = 1;
  670. setup_APIC_timer();
  671. }
  672. return;
  673. }
  674. apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
  675. "calibrating APIC timer ...\n");
  676. if (calibrate_APIC_clock()) {
  677. /* No broadcast on UP ! */
  678. if (num_possible_cpus() > 1)
  679. setup_APIC_timer();
  680. return;
  681. }
  682. /*
  683. * If nmi_watchdog is set to IO_APIC, we need the
  684. * PIT/HPET going. Otherwise register lapic as a dummy
  685. * device.
  686. */
  687. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
  688. /* Setup the lapic or request the broadcast */
  689. setup_APIC_timer();
  690. }
  691. void __cpuinit setup_secondary_APIC_clock(void)
  692. {
  693. setup_APIC_timer();
  694. }
  695. /*
  696. * The guts of the apic timer interrupt
  697. */
  698. static void local_apic_timer_interrupt(void)
  699. {
  700. int cpu = smp_processor_id();
  701. struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
  702. /*
  703. * Normally we should not be here till LAPIC has been initialized but
  704. * in some cases like kdump, its possible that there is a pending LAPIC
  705. * timer interrupt from previous kernel's context and is delivered in
  706. * new kernel the moment interrupts are enabled.
  707. *
  708. * Interrupts are enabled early and LAPIC is setup much later, hence
  709. * its possible that when we get here evt->event_handler is NULL.
  710. * Check for event_handler being NULL and discard the interrupt as
  711. * spurious.
  712. */
  713. if (!evt->event_handler) {
  714. pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
  715. /* Switch it off */
  716. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
  717. return;
  718. }
  719. /*
  720. * the NMI deadlock-detector uses this.
  721. */
  722. inc_irq_stat(apic_timer_irqs);
  723. evt->event_handler(evt);
  724. }
  725. /*
  726. * Local APIC timer interrupt. This is the most natural way for doing
  727. * local interrupts, but local timer interrupts can be emulated by
  728. * broadcast interrupts too. [in case the hw doesn't support APIC timers]
  729. *
  730. * [ if a single-CPU system runs an SMP kernel then we call the local
  731. * interrupt as well. Thus we cannot inline the local irq ... ]
  732. */
  733. void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
  734. {
  735. struct pt_regs *old_regs = set_irq_regs(regs);
  736. /*
  737. * NOTE! We'd better ACK the irq immediately,
  738. * because timer handling can be slow.
  739. */
  740. ack_APIC_irq();
  741. /*
  742. * update_process_times() expects us to have done irq_enter().
  743. * Besides, if we don't timer interrupts ignore the global
  744. * interrupt lock, which is the WrongThing (tm) to do.
  745. */
  746. exit_idle();
  747. irq_enter();
  748. local_apic_timer_interrupt();
  749. irq_exit();
  750. set_irq_regs(old_regs);
  751. }
  752. int setup_profiling_timer(unsigned int multiplier)
  753. {
  754. return -EINVAL;
  755. }
  756. /*
  757. * Local APIC start and shutdown
  758. */
  759. /**
  760. * clear_local_APIC - shutdown the local APIC
  761. *
  762. * This is called, when a CPU is disabled and before rebooting, so the state of
  763. * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
  764. * leftovers during boot.
  765. */
  766. void clear_local_APIC(void)
  767. {
  768. int maxlvt;
  769. u32 v;
  770. /* APIC hasn't been mapped yet */
  771. if (!x2apic_mode && !apic_phys)
  772. return;
  773. maxlvt = lapic_get_maxlvt();
  774. /*
  775. * Masking an LVT entry can trigger a local APIC error
  776. * if the vector is zero. Mask LVTERR first to prevent this.
  777. */
  778. if (maxlvt >= 3) {
  779. v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
  780. apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
  781. }
  782. /*
  783. * Careful: we have to set masks only first to deassert
  784. * any level-triggered sources.
  785. */
  786. v = apic_read(APIC_LVTT);
  787. apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
  788. v = apic_read(APIC_LVT0);
  789. apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
  790. v = apic_read(APIC_LVT1);
  791. apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
  792. if (maxlvt >= 4) {
  793. v = apic_read(APIC_LVTPC);
  794. apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
  795. }
  796. /* lets not touch this if we didn't frob it */
  797. #ifdef CONFIG_X86_THERMAL_VECTOR
  798. if (maxlvt >= 5) {
  799. v = apic_read(APIC_LVTTHMR);
  800. apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
  801. }
  802. #endif
  803. #ifdef CONFIG_X86_MCE_INTEL
  804. if (maxlvt >= 6) {
  805. v = apic_read(APIC_LVTCMCI);
  806. if (!(v & APIC_LVT_MASKED))
  807. apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
  808. }
  809. #endif
  810. /*
  811. * Clean APIC state for other OSs:
  812. */
  813. apic_write(APIC_LVTT, APIC_LVT_MASKED);
  814. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  815. apic_write(APIC_LVT1, APIC_LVT_MASKED);
  816. if (maxlvt >= 3)
  817. apic_write(APIC_LVTERR, APIC_LVT_MASKED);
  818. if (maxlvt >= 4)
  819. apic_write(APIC_LVTPC, APIC_LVT_MASKED);
  820. /* Integrated APIC (!82489DX) ? */
  821. if (lapic_is_integrated()) {
  822. if (maxlvt > 3)
  823. /* Clear ESR due to Pentium errata 3AP and 11AP */
  824. apic_write(APIC_ESR, 0);
  825. apic_read(APIC_ESR);
  826. }
  827. }
  828. /**
  829. * disable_local_APIC - clear and disable the local APIC
  830. */
  831. void disable_local_APIC(void)
  832. {
  833. unsigned int value;
  834. /* APIC hasn't been mapped yet */
  835. if (!x2apic_mode && !apic_phys)
  836. return;
  837. clear_local_APIC();
  838. /*
  839. * Disable APIC (implies clearing of registers
  840. * for 82489DX!).
  841. */
  842. value = apic_read(APIC_SPIV);
  843. value &= ~APIC_SPIV_APIC_ENABLED;
  844. apic_write(APIC_SPIV, value);
  845. #ifdef CONFIG_X86_32
  846. /*
  847. * When LAPIC was disabled by the BIOS and enabled by the kernel,
  848. * restore the disabled state.
  849. */
  850. if (enabled_via_apicbase) {
  851. unsigned int l, h;
  852. rdmsr(MSR_IA32_APICBASE, l, h);
  853. l &= ~MSR_IA32_APICBASE_ENABLE;
  854. wrmsr(MSR_IA32_APICBASE, l, h);
  855. }
  856. #endif
  857. }
  858. /*
  859. * If Linux enabled the LAPIC against the BIOS default disable it down before
  860. * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
  861. * not power-off. Additionally clear all LVT entries before disable_local_APIC
  862. * for the case where Linux didn't enable the LAPIC.
  863. */
  864. void lapic_shutdown(void)
  865. {
  866. unsigned long flags;
  867. if (!cpu_has_apic && !apic_from_smp_config())
  868. return;
  869. local_irq_save(flags);
  870. #ifdef CONFIG_X86_32
  871. if (!enabled_via_apicbase)
  872. clear_local_APIC();
  873. else
  874. #endif
  875. disable_local_APIC();
  876. local_irq_restore(flags);
  877. }
  878. /*
  879. * This is to verify that we're looking at a real local APIC.
  880. * Check these against your board if the CPUs aren't getting
  881. * started for no apparent reason.
  882. */
  883. int __init verify_local_APIC(void)
  884. {
  885. unsigned int reg0, reg1;
  886. /*
  887. * The version register is read-only in a real APIC.
  888. */
  889. reg0 = apic_read(APIC_LVR);
  890. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
  891. apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
  892. reg1 = apic_read(APIC_LVR);
  893. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
  894. /*
  895. * The two version reads above should print the same
  896. * numbers. If the second one is different, then we
  897. * poke at a non-APIC.
  898. */
  899. if (reg1 != reg0)
  900. return 0;
  901. /*
  902. * Check if the version looks reasonably.
  903. */
  904. reg1 = GET_APIC_VERSION(reg0);
  905. if (reg1 == 0x00 || reg1 == 0xff)
  906. return 0;
  907. reg1 = lapic_get_maxlvt();
  908. if (reg1 < 0x02 || reg1 == 0xff)
  909. return 0;
  910. /*
  911. * The ID register is read/write in a real APIC.
  912. */
  913. reg0 = apic_read(APIC_ID);
  914. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
  915. apic_write(APIC_ID, reg0 ^ apic->apic_id_mask);
  916. reg1 = apic_read(APIC_ID);
  917. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
  918. apic_write(APIC_ID, reg0);
  919. if (reg1 != (reg0 ^ apic->apic_id_mask))
  920. return 0;
  921. /*
  922. * The next two are just to see if we have sane values.
  923. * They're only really relevant if we're in Virtual Wire
  924. * compatibility mode, but most boxes are anymore.
  925. */
  926. reg0 = apic_read(APIC_LVT0);
  927. apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
  928. reg1 = apic_read(APIC_LVT1);
  929. apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
  930. return 1;
  931. }
  932. /**
  933. * sync_Arb_IDs - synchronize APIC bus arbitration IDs
  934. */
  935. void __init sync_Arb_IDs(void)
  936. {
  937. /*
  938. * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
  939. * needed on AMD.
  940. */
  941. if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
  942. return;
  943. /*
  944. * Wait for idle.
  945. */
  946. apic_wait_icr_idle();
  947. apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
  948. apic_write(APIC_ICR, APIC_DEST_ALLINC |
  949. APIC_INT_LEVELTRIG | APIC_DM_INIT);
  950. }
  951. /*
  952. * An initial setup of the virtual wire mode.
  953. */
  954. void __init init_bsp_APIC(void)
  955. {
  956. unsigned int value;
  957. /*
  958. * Don't do the setup now if we have a SMP BIOS as the
  959. * through-I/O-APIC virtual wire mode might be active.
  960. */
  961. if (smp_found_config || !cpu_has_apic)
  962. return;
  963. /*
  964. * Do not trust the local APIC being empty at bootup.
  965. */
  966. clear_local_APIC();
  967. /*
  968. * Enable APIC.
  969. */
  970. value = apic_read(APIC_SPIV);
  971. value &= ~APIC_VECTOR_MASK;
  972. value |= APIC_SPIV_APIC_ENABLED;
  973. #ifdef CONFIG_X86_32
  974. /* This bit is reserved on P4/Xeon and should be cleared */
  975. if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
  976. (boot_cpu_data.x86 == 15))
  977. value &= ~APIC_SPIV_FOCUS_DISABLED;
  978. else
  979. #endif
  980. value |= APIC_SPIV_FOCUS_DISABLED;
  981. value |= SPURIOUS_APIC_VECTOR;
  982. apic_write(APIC_SPIV, value);
  983. /*
  984. * Set up the virtual wire mode.
  985. */
  986. apic_write(APIC_LVT0, APIC_DM_EXTINT);
  987. value = APIC_DM_NMI;
  988. if (!lapic_is_integrated()) /* 82489DX */
  989. value |= APIC_LVT_LEVEL_TRIGGER;
  990. apic_write(APIC_LVT1, value);
  991. }
  992. static void __cpuinit lapic_setup_esr(void)
  993. {
  994. unsigned int oldvalue, value, maxlvt;
  995. if (!lapic_is_integrated()) {
  996. pr_info("No ESR for 82489DX.\n");
  997. return;
  998. }
  999. if (apic->disable_esr) {
  1000. /*
  1001. * Something untraceable is creating bad interrupts on
  1002. * secondary quads ... for the moment, just leave the
  1003. * ESR disabled - we can't do anything useful with the
  1004. * errors anyway - mbligh
  1005. */
  1006. pr_info("Leaving ESR disabled.\n");
  1007. return;
  1008. }
  1009. maxlvt = lapic_get_maxlvt();
  1010. if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
  1011. apic_write(APIC_ESR, 0);
  1012. oldvalue = apic_read(APIC_ESR);
  1013. /* enables sending errors */
  1014. value = ERROR_APIC_VECTOR;
  1015. apic_write(APIC_LVTERR, value);
  1016. /*
  1017. * spec says clear errors after enabling vector.
  1018. */
  1019. if (maxlvt > 3)
  1020. apic_write(APIC_ESR, 0);
  1021. value = apic_read(APIC_ESR);
  1022. if (value != oldvalue)
  1023. apic_printk(APIC_VERBOSE, "ESR value before enabling "
  1024. "vector: 0x%08x after: 0x%08x\n",
  1025. oldvalue, value);
  1026. }
  1027. /**
  1028. * setup_local_APIC - setup the local APIC
  1029. *
  1030. * Used to setup local APIC while initializing BSP or bringin up APs.
  1031. * Always called with preemption disabled.
  1032. */
  1033. void __cpuinit setup_local_APIC(void)
  1034. {
  1035. int cpu = smp_processor_id();
  1036. unsigned int value, queued;
  1037. int i, j, acked = 0;
  1038. unsigned long long tsc = 0, ntsc;
  1039. long long max_loops = cpu_khz;
  1040. if (cpu_has_tsc)
  1041. rdtscll(tsc);
  1042. if (disable_apic) {
  1043. disable_ioapic_support();
  1044. return;
  1045. }
  1046. #ifdef CONFIG_X86_32
  1047. /* Pound the ESR really hard over the head with a big hammer - mbligh */
  1048. if (lapic_is_integrated() && apic->disable_esr) {
  1049. apic_write(APIC_ESR, 0);
  1050. apic_write(APIC_ESR, 0);
  1051. apic_write(APIC_ESR, 0);
  1052. apic_write(APIC_ESR, 0);
  1053. }
  1054. #endif
  1055. perf_events_lapic_init();
  1056. /*
  1057. * Double-check whether this APIC is really registered.
  1058. * This is meaningless in clustered apic mode, so we skip it.
  1059. */
  1060. BUG_ON(!apic->apic_id_registered());
  1061. /*
  1062. * Intel recommends to set DFR, LDR and TPR before enabling
  1063. * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
  1064. * document number 292116). So here it goes...
  1065. */
  1066. apic->init_apic_ldr();
  1067. #ifdef CONFIG_X86_32
  1068. /*
  1069. * APIC LDR is initialized. If logical_apicid mapping was
  1070. * initialized during get_smp_config(), make sure it matches the
  1071. * actual value.
  1072. */
  1073. i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
  1074. WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
  1075. /* always use the value from LDR */
  1076. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1077. logical_smp_processor_id();
  1078. /*
  1079. * Some NUMA implementations (NUMAQ) don't initialize apicid to
  1080. * node mapping during NUMA init. Now that logical apicid is
  1081. * guaranteed to be known, give it another chance. This is already
  1082. * a bit too late - percpu allocation has already happened without
  1083. * proper NUMA affinity.
  1084. */
  1085. if (apic->x86_32_numa_cpu_node)
  1086. set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu),
  1087. apic->x86_32_numa_cpu_node(cpu));
  1088. #endif
  1089. /*
  1090. * Set Task Priority to 'accept all'. We never change this
  1091. * later on.
  1092. */
  1093. value = apic_read(APIC_TASKPRI);
  1094. value &= ~APIC_TPRI_MASK;
  1095. apic_write(APIC_TASKPRI, value);
  1096. /*
  1097. * After a crash, we no longer service the interrupts and a pending
  1098. * interrupt from previous kernel might still have ISR bit set.
  1099. *
  1100. * Most probably by now CPU has serviced that pending interrupt and
  1101. * it might not have done the ack_APIC_irq() because it thought,
  1102. * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
  1103. * does not clear the ISR bit and cpu thinks it has already serivced
  1104. * the interrupt. Hence a vector might get locked. It was noticed
  1105. * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
  1106. */
  1107. do {
  1108. queued = 0;
  1109. for (i = APIC_ISR_NR - 1; i >= 0; i--)
  1110. queued |= apic_read(APIC_IRR + i*0x10);
  1111. for (i = APIC_ISR_NR - 1; i >= 0; i--) {
  1112. value = apic_read(APIC_ISR + i*0x10);
  1113. for (j = 31; j >= 0; j--) {
  1114. if (value & (1<<j)) {
  1115. ack_APIC_irq();
  1116. acked++;
  1117. }
  1118. }
  1119. }
  1120. if (acked > 256) {
  1121. printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
  1122. acked);
  1123. break;
  1124. }
  1125. if (cpu_has_tsc) {
  1126. rdtscll(ntsc);
  1127. max_loops = (cpu_khz << 10) - (ntsc - tsc);
  1128. } else
  1129. max_loops--;
  1130. } while (queued && max_loops > 0);
  1131. WARN_ON(max_loops <= 0);
  1132. /*
  1133. * Now that we are all set up, enable the APIC
  1134. */
  1135. value = apic_read(APIC_SPIV);
  1136. value &= ~APIC_VECTOR_MASK;
  1137. /*
  1138. * Enable APIC
  1139. */
  1140. value |= APIC_SPIV_APIC_ENABLED;
  1141. #ifdef CONFIG_X86_32
  1142. /*
  1143. * Some unknown Intel IO/APIC (or APIC) errata is biting us with
  1144. * certain networking cards. If high frequency interrupts are
  1145. * happening on a particular IOAPIC pin, plus the IOAPIC routing
  1146. * entry is masked/unmasked at a high rate as well then sooner or
  1147. * later IOAPIC line gets 'stuck', no more interrupts are received
  1148. * from the device. If focus CPU is disabled then the hang goes
  1149. * away, oh well :-(
  1150. *
  1151. * [ This bug can be reproduced easily with a level-triggered
  1152. * PCI Ne2000 networking cards and PII/PIII processors, dual
  1153. * BX chipset. ]
  1154. */
  1155. /*
  1156. * Actually disabling the focus CPU check just makes the hang less
  1157. * frequent as it makes the interrupt distributon model be more
  1158. * like LRU than MRU (the short-term load is more even across CPUs).
  1159. * See also the comment in end_level_ioapic_irq(). --macro
  1160. */
  1161. /*
  1162. * - enable focus processor (bit==0)
  1163. * - 64bit mode always use processor focus
  1164. * so no need to set it
  1165. */
  1166. value &= ~APIC_SPIV_FOCUS_DISABLED;
  1167. #endif
  1168. /*
  1169. * Set spurious IRQ vector
  1170. */
  1171. value |= SPURIOUS_APIC_VECTOR;
  1172. apic_write(APIC_SPIV, value);
  1173. /*
  1174. * Set up LVT0, LVT1:
  1175. *
  1176. * set up through-local-APIC on the BP's LINT0. This is not
  1177. * strictly necessary in pure symmetric-IO mode, but sometimes
  1178. * we delegate interrupts to the 8259A.
  1179. */
  1180. /*
  1181. * TODO: set up through-local-APIC from through-I/O-APIC? --macro
  1182. */
  1183. value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
  1184. if (!cpu && (pic_mode || !value)) {
  1185. value = APIC_DM_EXTINT;
  1186. apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
  1187. } else {
  1188. value = APIC_DM_EXTINT | APIC_LVT_MASKED;
  1189. apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
  1190. }
  1191. apic_write(APIC_LVT0, value);
  1192. /*
  1193. * only the BP should see the LINT1 NMI signal, obviously.
  1194. */
  1195. if (!cpu)
  1196. value = APIC_DM_NMI;
  1197. else
  1198. value = APIC_DM_NMI | APIC_LVT_MASKED;
  1199. if (!lapic_is_integrated()) /* 82489DX */
  1200. value |= APIC_LVT_LEVEL_TRIGGER;
  1201. apic_write(APIC_LVT1, value);
  1202. #ifdef CONFIG_X86_MCE_INTEL
  1203. /* Recheck CMCI information after local APIC is up on CPU #0 */
  1204. if (!cpu)
  1205. cmci_recheck();
  1206. #endif
  1207. }
  1208. void __cpuinit end_local_APIC_setup(void)
  1209. {
  1210. lapic_setup_esr();
  1211. #ifdef CONFIG_X86_32
  1212. {
  1213. unsigned int value;
  1214. /* Disable the local apic timer */
  1215. value = apic_read(APIC_LVTT);
  1216. value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  1217. apic_write(APIC_LVTT, value);
  1218. }
  1219. #endif
  1220. apic_pm_activate();
  1221. }
  1222. void __init bsp_end_local_APIC_setup(void)
  1223. {
  1224. end_local_APIC_setup();
  1225. /*
  1226. * Now that local APIC setup is completed for BP, configure the fault
  1227. * handling for interrupt remapping.
  1228. */
  1229. if (intr_remapping_enabled)
  1230. enable_drhd_fault_handling();
  1231. }
  1232. #ifdef CONFIG_X86_X2APIC
  1233. void check_x2apic(void)
  1234. {
  1235. if (x2apic_enabled()) {
  1236. pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
  1237. x2apic_preenabled = x2apic_mode = 1;
  1238. }
  1239. }
  1240. void enable_x2apic(void)
  1241. {
  1242. int msr, msr2;
  1243. if (!x2apic_mode)
  1244. return;
  1245. rdmsr(MSR_IA32_APICBASE, msr, msr2);
  1246. if (!(msr & X2APIC_ENABLE)) {
  1247. printk_once(KERN_INFO "Enabling x2apic\n");
  1248. wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2);
  1249. }
  1250. }
  1251. #endif /* CONFIG_X86_X2APIC */
  1252. int __init enable_IR(void)
  1253. {
  1254. #ifdef CONFIG_IRQ_REMAP
  1255. if (!intr_remapping_supported()) {
  1256. pr_debug("intr-remapping not supported\n");
  1257. return -1;
  1258. }
  1259. if (!x2apic_preenabled && skip_ioapic_setup) {
  1260. pr_info("Skipped enabling intr-remap because of skipping "
  1261. "io-apic setup\n");
  1262. return -1;
  1263. }
  1264. return enable_intr_remapping();
  1265. #endif
  1266. return -1;
  1267. }
  1268. void __init enable_IR_x2apic(void)
  1269. {
  1270. unsigned long flags;
  1271. int ret, x2apic_enabled = 0;
  1272. int dmar_table_init_ret;
  1273. dmar_table_init_ret = dmar_table_init();
  1274. if (dmar_table_init_ret && !x2apic_supported())
  1275. return;
  1276. ret = save_ioapic_entries();
  1277. if (ret) {
  1278. pr_info("Saving IO-APIC state failed: %d\n", ret);
  1279. goto out;
  1280. }
  1281. local_irq_save(flags);
  1282. legacy_pic->mask_all();
  1283. mask_ioapic_entries();
  1284. if (dmar_table_init_ret)
  1285. ret = -1;
  1286. else
  1287. ret = enable_IR();
  1288. if (ret < 0) {
  1289. /* IR is required if there is APIC ID > 255 even when running
  1290. * under KVM
  1291. */
  1292. if (max_physical_apicid > 255 ||
  1293. !hypervisor_x2apic_available())
  1294. goto nox2apic;
  1295. /*
  1296. * without IR all CPUs can be addressed by IOAPIC/MSI
  1297. * only in physical mode
  1298. */
  1299. x2apic_force_phys();
  1300. }
  1301. if (ret == IRQ_REMAP_XAPIC_MODE)
  1302. goto nox2apic;
  1303. x2apic_enabled = 1;
  1304. if (x2apic_supported() && !x2apic_mode) {
  1305. x2apic_mode = 1;
  1306. enable_x2apic();
  1307. pr_info("Enabled x2apic\n");
  1308. }
  1309. nox2apic:
  1310. if (ret < 0) /* IR enabling failed */
  1311. restore_ioapic_entries();
  1312. legacy_pic->restore_mask();
  1313. local_irq_restore(flags);
  1314. out:
  1315. if (x2apic_enabled || !x2apic_supported())
  1316. return;
  1317. if (x2apic_preenabled)
  1318. panic("x2apic: enabled by BIOS but kernel init failed.");
  1319. else if (ret == IRQ_REMAP_XAPIC_MODE)
  1320. pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n");
  1321. else if (ret < 0)
  1322. pr_info("x2apic not enabled, IRQ remapping init failed\n");
  1323. }
  1324. #ifdef CONFIG_X86_64
  1325. /*
  1326. * Detect and enable local APICs on non-SMP boards.
  1327. * Original code written by Keir Fraser.
  1328. * On AMD64 we trust the BIOS - if it says no APIC it is likely
  1329. * not correctly set up (usually the APIC timer won't work etc.)
  1330. */
  1331. static int __init detect_init_APIC(void)
  1332. {
  1333. if (!cpu_has_apic) {
  1334. pr_info("No local APIC present\n");
  1335. return -1;
  1336. }
  1337. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1338. return 0;
  1339. }
  1340. #else
  1341. static int __init apic_verify(void)
  1342. {
  1343. u32 features, h, l;
  1344. /*
  1345. * The APIC feature bit should now be enabled
  1346. * in `cpuid'
  1347. */
  1348. features = cpuid_edx(1);
  1349. if (!(features & (1 << X86_FEATURE_APIC))) {
  1350. pr_warning("Could not enable APIC!\n");
  1351. return -1;
  1352. }
  1353. set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
  1354. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1355. /* The BIOS may have set up the APIC at some other address */
  1356. rdmsr(MSR_IA32_APICBASE, l, h);
  1357. if (l & MSR_IA32_APICBASE_ENABLE)
  1358. mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
  1359. pr_info("Found and enabled local APIC!\n");
  1360. return 0;
  1361. }
  1362. int __init apic_force_enable(unsigned long addr)
  1363. {
  1364. u32 h, l;
  1365. if (disable_apic)
  1366. return -1;
  1367. /*
  1368. * Some BIOSes disable the local APIC in the APIC_BASE
  1369. * MSR. This can only be done in software for Intel P6 or later
  1370. * and AMD K7 (Model > 1) or later.
  1371. */
  1372. rdmsr(MSR_IA32_APICBASE, l, h);
  1373. if (!(l & MSR_IA32_APICBASE_ENABLE)) {
  1374. pr_info("Local APIC disabled by BIOS -- reenabling.\n");
  1375. l &= ~MSR_IA32_APICBASE_BASE;
  1376. l |= MSR_IA32_APICBASE_ENABLE | addr;
  1377. wrmsr(MSR_IA32_APICBASE, l, h);
  1378. enabled_via_apicbase = 1;
  1379. }
  1380. return apic_verify();
  1381. }
  1382. /*
  1383. * Detect and initialize APIC
  1384. */
  1385. static int __init detect_init_APIC(void)
  1386. {
  1387. /* Disabled by kernel option? */
  1388. if (disable_apic)
  1389. return -1;
  1390. switch (boot_cpu_data.x86_vendor) {
  1391. case X86_VENDOR_AMD:
  1392. if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
  1393. (boot_cpu_data.x86 >= 15))
  1394. break;
  1395. goto no_apic;
  1396. case X86_VENDOR_INTEL:
  1397. if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
  1398. (boot_cpu_data.x86 == 5 && cpu_has_apic))
  1399. break;
  1400. goto no_apic;
  1401. default:
  1402. goto no_apic;
  1403. }
  1404. if (!cpu_has_apic) {
  1405. /*
  1406. * Over-ride BIOS and try to enable the local APIC only if
  1407. * "lapic" specified.
  1408. */
  1409. if (!force_enable_local_apic) {
  1410. pr_info("Local APIC disabled by BIOS -- "
  1411. "you can enable it with \"lapic\"\n");
  1412. return -1;
  1413. }
  1414. if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
  1415. return -1;
  1416. } else {
  1417. if (apic_verify())
  1418. return -1;
  1419. }
  1420. apic_pm_activate();
  1421. return 0;
  1422. no_apic:
  1423. pr_info("No local APIC present or hardware disabled\n");
  1424. return -1;
  1425. }
  1426. #endif
  1427. /**
  1428. * init_apic_mappings - initialize APIC mappings
  1429. */
  1430. void __init init_apic_mappings(void)
  1431. {
  1432. unsigned int new_apicid;
  1433. if (x2apic_mode) {
  1434. boot_cpu_physical_apicid = read_apic_id();
  1435. return;
  1436. }
  1437. /* If no local APIC can be found return early */
  1438. if (!smp_found_config && detect_init_APIC()) {
  1439. /* lets NOP'ify apic operations */
  1440. pr_info("APIC: disable apic facility\n");
  1441. apic_disable();
  1442. } else {
  1443. apic_phys = mp_lapic_addr;
  1444. /*
  1445. * acpi lapic path already maps that address in
  1446. * acpi_register_lapic_address()
  1447. */
  1448. if (!acpi_lapic && !smp_found_config)
  1449. register_lapic_address(apic_phys);
  1450. }
  1451. /*
  1452. * Fetch the APIC ID of the BSP in case we have a
  1453. * default configuration (or the MP table is broken).
  1454. */
  1455. new_apicid = read_apic_id();
  1456. if (boot_cpu_physical_apicid != new_apicid) {
  1457. boot_cpu_physical_apicid = new_apicid;
  1458. /*
  1459. * yeah -- we lie about apic_version
  1460. * in case if apic was disabled via boot option
  1461. * but it's not a problem for SMP compiled kernel
  1462. * since smp_sanity_check is prepared for such a case
  1463. * and disable smp mode
  1464. */
  1465. apic_version[new_apicid] =
  1466. GET_APIC_VERSION(apic_read(APIC_LVR));
  1467. }
  1468. }
  1469. void __init register_lapic_address(unsigned long address)
  1470. {
  1471. mp_lapic_addr = address;
  1472. if (!x2apic_mode) {
  1473. set_fixmap_nocache(FIX_APIC_BASE, address);
  1474. apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
  1475. APIC_BASE, mp_lapic_addr);
  1476. }
  1477. if (boot_cpu_physical_apicid == -1U) {
  1478. boot_cpu_physical_apicid = read_apic_id();
  1479. apic_version[boot_cpu_physical_apicid] =
  1480. GET_APIC_VERSION(apic_read(APIC_LVR));
  1481. }
  1482. }
  1483. /*
  1484. * This initializes the IO-APIC and APIC hardware if this is
  1485. * a UP kernel.
  1486. */
  1487. int apic_version[MAX_LOCAL_APIC];
  1488. int __init APIC_init_uniprocessor(void)
  1489. {
  1490. if (disable_apic) {
  1491. pr_info("Apic disabled\n");
  1492. return -1;
  1493. }
  1494. #ifdef CONFIG_X86_64
  1495. if (!cpu_has_apic) {
  1496. disable_apic = 1;
  1497. pr_info("Apic disabled by BIOS\n");
  1498. return -1;
  1499. }
  1500. #else
  1501. if (!smp_found_config && !cpu_has_apic)
  1502. return -1;
  1503. /*
  1504. * Complain if the BIOS pretends there is one.
  1505. */
  1506. if (!cpu_has_apic &&
  1507. APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
  1508. pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
  1509. boot_cpu_physical_apicid);
  1510. return -1;
  1511. }
  1512. #endif
  1513. default_setup_apic_routing();
  1514. verify_local_APIC();
  1515. connect_bsp_APIC();
  1516. #ifdef CONFIG_X86_64
  1517. apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
  1518. #else
  1519. /*
  1520. * Hack: In case of kdump, after a crash, kernel might be booting
  1521. * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
  1522. * might be zero if read from MP tables. Get it from LAPIC.
  1523. */
  1524. # ifdef CONFIG_CRASH_DUMP
  1525. boot_cpu_physical_apicid = read_apic_id();
  1526. # endif
  1527. #endif
  1528. physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
  1529. setup_local_APIC();
  1530. #ifdef CONFIG_X86_IO_APIC
  1531. /*
  1532. * Now enable IO-APICs, actually call clear_IO_APIC
  1533. * We need clear_IO_APIC before enabling error vector
  1534. */
  1535. if (!skip_ioapic_setup && nr_ioapics)
  1536. enable_IO_APIC();
  1537. #endif
  1538. bsp_end_local_APIC_setup();
  1539. #ifdef CONFIG_X86_IO_APIC
  1540. if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
  1541. setup_IO_APIC();
  1542. else {
  1543. nr_ioapics = 0;
  1544. }
  1545. #endif
  1546. x86_init.timers.setup_percpu_clockev();
  1547. return 0;
  1548. }
  1549. /*
  1550. * Local APIC interrupts
  1551. */
  1552. /*
  1553. * This interrupt should _never_ happen with our APIC/SMP architecture
  1554. */
  1555. void smp_spurious_interrupt(struct pt_regs *regs)
  1556. {
  1557. u32 v;
  1558. exit_idle();
  1559. irq_enter();
  1560. /*
  1561. * Check if this really is a spurious interrupt and ACK it
  1562. * if it is a vectored one. Just in case...
  1563. * Spurious interrupts should not be ACKed.
  1564. */
  1565. v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
  1566. if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
  1567. ack_APIC_irq();
  1568. inc_irq_stat(irq_spurious_count);
  1569. /* see sw-dev-man vol 3, chapter 7.4.13.5 */
  1570. pr_info("spurious APIC interrupt on CPU#%d, "
  1571. "should never happen.\n", smp_processor_id());
  1572. irq_exit();
  1573. }
  1574. /*
  1575. * This interrupt should never happen with our APIC/SMP architecture
  1576. */
  1577. void smp_error_interrupt(struct pt_regs *regs)
  1578. {
  1579. u32 v0, v1;
  1580. u32 i = 0;
  1581. static const char * const error_interrupt_reason[] = {
  1582. "Send CS error", /* APIC Error Bit 0 */
  1583. "Receive CS error", /* APIC Error Bit 1 */
  1584. "Send accept error", /* APIC Error Bit 2 */
  1585. "Receive accept error", /* APIC Error Bit 3 */
  1586. "Redirectable IPI", /* APIC Error Bit 4 */
  1587. "Send illegal vector", /* APIC Error Bit 5 */
  1588. "Received illegal vector", /* APIC Error Bit 6 */
  1589. "Illegal register address", /* APIC Error Bit 7 */
  1590. };
  1591. exit_idle();
  1592. irq_enter();
  1593. /* First tickle the hardware, only then report what went on. -- REW */
  1594. v0 = apic_read(APIC_ESR);
  1595. apic_write(APIC_ESR, 0);
  1596. v1 = apic_read(APIC_ESR);
  1597. ack_APIC_irq();
  1598. atomic_inc(&irq_err_count);
  1599. apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x(%02x)",
  1600. smp_processor_id(), v0 , v1);
  1601. v1 = v1 & 0xff;
  1602. while (v1) {
  1603. if (v1 & 0x1)
  1604. apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
  1605. i++;
  1606. v1 >>= 1;
  1607. };
  1608. apic_printk(APIC_DEBUG, KERN_CONT "\n");
  1609. irq_exit();
  1610. }
  1611. /**
  1612. * connect_bsp_APIC - attach the APIC to the interrupt system
  1613. */
  1614. void __init connect_bsp_APIC(void)
  1615. {
  1616. #ifdef CONFIG_X86_32
  1617. if (pic_mode) {
  1618. /*
  1619. * Do not trust the local APIC being empty at bootup.
  1620. */
  1621. clear_local_APIC();
  1622. /*
  1623. * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
  1624. * local APIC to INT and NMI lines.
  1625. */
  1626. apic_printk(APIC_VERBOSE, "leaving PIC mode, "
  1627. "enabling APIC mode.\n");
  1628. imcr_pic_to_apic();
  1629. }
  1630. #endif
  1631. if (apic->enable_apic_mode)
  1632. apic->enable_apic_mode();
  1633. }
  1634. /**
  1635. * disconnect_bsp_APIC - detach the APIC from the interrupt system
  1636. * @virt_wire_setup: indicates, whether virtual wire mode is selected
  1637. *
  1638. * Virtual wire mode is necessary to deliver legacy interrupts even when the
  1639. * APIC is disabled.
  1640. */
  1641. void disconnect_bsp_APIC(int virt_wire_setup)
  1642. {
  1643. unsigned int value;
  1644. #ifdef CONFIG_X86_32
  1645. if (pic_mode) {
  1646. /*
  1647. * Put the board back into PIC mode (has an effect only on
  1648. * certain older boards). Note that APIC interrupts, including
  1649. * IPIs, won't work beyond this point! The only exception are
  1650. * INIT IPIs.
  1651. */
  1652. apic_printk(APIC_VERBOSE, "disabling APIC mode, "
  1653. "entering PIC mode.\n");
  1654. imcr_apic_to_pic();
  1655. return;
  1656. }
  1657. #endif
  1658. /* Go back to Virtual Wire compatibility mode */
  1659. /* For the spurious interrupt use vector F, and enable it */
  1660. value = apic_read(APIC_SPIV);
  1661. value &= ~APIC_VECTOR_MASK;
  1662. value |= APIC_SPIV_APIC_ENABLED;
  1663. value |= 0xf;
  1664. apic_write(APIC_SPIV, value);
  1665. if (!virt_wire_setup) {
  1666. /*
  1667. * For LVT0 make it edge triggered, active high,
  1668. * external and enabled
  1669. */
  1670. value = apic_read(APIC_LVT0);
  1671. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1672. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1673. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1674. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1675. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
  1676. apic_write(APIC_LVT0, value);
  1677. } else {
  1678. /* Disable LVT0 */
  1679. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  1680. }
  1681. /*
  1682. * For LVT1 make it edge triggered, active high,
  1683. * nmi and enabled
  1684. */
  1685. value = apic_read(APIC_LVT1);
  1686. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1687. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1688. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1689. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1690. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
  1691. apic_write(APIC_LVT1, value);
  1692. }
  1693. void __cpuinit generic_processor_info(int apicid, int version)
  1694. {
  1695. int cpu, max = nr_cpu_ids;
  1696. bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
  1697. phys_cpu_present_map);
  1698. /*
  1699. * If boot cpu has not been detected yet, then only allow upto
  1700. * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
  1701. */
  1702. if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
  1703. apicid != boot_cpu_physical_apicid) {
  1704. int thiscpu = max + disabled_cpus - 1;
  1705. pr_warning(
  1706. "ACPI: NR_CPUS/possible_cpus limit of %i almost"
  1707. " reached. Keeping one slot for boot cpu."
  1708. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1709. disabled_cpus++;
  1710. return;
  1711. }
  1712. if (num_processors >= nr_cpu_ids) {
  1713. int thiscpu = max + disabled_cpus;
  1714. pr_warning(
  1715. "ACPI: NR_CPUS/possible_cpus limit of %i reached."
  1716. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1717. disabled_cpus++;
  1718. return;
  1719. }
  1720. num_processors++;
  1721. if (apicid == boot_cpu_physical_apicid) {
  1722. /*
  1723. * x86_bios_cpu_apicid is required to have processors listed
  1724. * in same order as logical cpu numbers. Hence the first
  1725. * entry is BSP, and so on.
  1726. * boot_cpu_init() already hold bit 0 in cpu_present_mask
  1727. * for BSP.
  1728. */
  1729. cpu = 0;
  1730. } else
  1731. cpu = cpumask_next_zero(-1, cpu_present_mask);
  1732. /*
  1733. * Validate version
  1734. */
  1735. if (version == 0x0) {
  1736. pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
  1737. cpu, apicid);
  1738. version = 0x10;
  1739. }
  1740. apic_version[apicid] = version;
  1741. if (version != apic_version[boot_cpu_physical_apicid]) {
  1742. pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
  1743. apic_version[boot_cpu_physical_apicid], cpu, version);
  1744. }
  1745. physid_set(apicid, phys_cpu_present_map);
  1746. if (apicid > max_physical_apicid)
  1747. max_physical_apicid = apicid;
  1748. #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
  1749. early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
  1750. early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
  1751. #endif
  1752. #ifdef CONFIG_X86_32
  1753. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1754. apic->x86_32_early_logical_apicid(cpu);
  1755. #endif
  1756. set_cpu_possible(cpu, true);
  1757. set_cpu_present(cpu, true);
  1758. }
  1759. int hard_smp_processor_id(void)
  1760. {
  1761. return read_apic_id();
  1762. }
  1763. void default_init_apic_ldr(void)
  1764. {
  1765. unsigned long val;
  1766. apic_write(APIC_DFR, APIC_DFR_VALUE);
  1767. val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
  1768. val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
  1769. apic_write(APIC_LDR, val);
  1770. }
  1771. /*
  1772. * Power management
  1773. */
  1774. #ifdef CONFIG_PM
  1775. static struct {
  1776. /*
  1777. * 'active' is true if the local APIC was enabled by us and
  1778. * not the BIOS; this signifies that we are also responsible
  1779. * for disabling it before entering apm/acpi suspend
  1780. */
  1781. int active;
  1782. /* r/w apic fields */
  1783. unsigned int apic_id;
  1784. unsigned int apic_taskpri;
  1785. unsigned int apic_ldr;
  1786. unsigned int apic_dfr;
  1787. unsigned int apic_spiv;
  1788. unsigned int apic_lvtt;
  1789. unsigned int apic_lvtpc;
  1790. unsigned int apic_lvt0;
  1791. unsigned int apic_lvt1;
  1792. unsigned int apic_lvterr;
  1793. unsigned int apic_tmict;
  1794. unsigned int apic_tdcr;
  1795. unsigned int apic_thmr;
  1796. } apic_pm_state;
  1797. static int lapic_suspend(void)
  1798. {
  1799. unsigned long flags;
  1800. int maxlvt;
  1801. if (!apic_pm_state.active)
  1802. return 0;
  1803. maxlvt = lapic_get_maxlvt();
  1804. apic_pm_state.apic_id = apic_read(APIC_ID);
  1805. apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
  1806. apic_pm_state.apic_ldr = apic_read(APIC_LDR);
  1807. apic_pm_state.apic_dfr = apic_read(APIC_DFR);
  1808. apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
  1809. apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
  1810. if (maxlvt >= 4)
  1811. apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
  1812. apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
  1813. apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
  1814. apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
  1815. apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
  1816. apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
  1817. #ifdef CONFIG_X86_THERMAL_VECTOR
  1818. if (maxlvt >= 5)
  1819. apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
  1820. #endif
  1821. local_irq_save(flags);
  1822. disable_local_APIC();
  1823. if (intr_remapping_enabled)
  1824. disable_intr_remapping();
  1825. local_irq_restore(flags);
  1826. return 0;
  1827. }
  1828. static void lapic_resume(void)
  1829. {
  1830. unsigned int l, h;
  1831. unsigned long flags;
  1832. int maxlvt;
  1833. if (!apic_pm_state.active)
  1834. return;
  1835. local_irq_save(flags);
  1836. if (intr_remapping_enabled) {
  1837. /*
  1838. * IO-APIC and PIC have their own resume routines.
  1839. * We just mask them here to make sure the interrupt
  1840. * subsystem is completely quiet while we enable x2apic
  1841. * and interrupt-remapping.
  1842. */
  1843. mask_ioapic_entries();
  1844. legacy_pic->mask_all();
  1845. }
  1846. if (x2apic_mode)
  1847. enable_x2apic();
  1848. else {
  1849. /*
  1850. * Make sure the APICBASE points to the right address
  1851. *
  1852. * FIXME! This will be wrong if we ever support suspend on
  1853. * SMP! We'll need to do this as part of the CPU restore!
  1854. */
  1855. rdmsr(MSR_IA32_APICBASE, l, h);
  1856. l &= ~MSR_IA32_APICBASE_BASE;
  1857. l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
  1858. wrmsr(MSR_IA32_APICBASE, l, h);
  1859. }
  1860. maxlvt = lapic_get_maxlvt();
  1861. apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
  1862. apic_write(APIC_ID, apic_pm_state.apic_id);
  1863. apic_write(APIC_DFR, apic_pm_state.apic_dfr);
  1864. apic_write(APIC_LDR, apic_pm_state.apic_ldr);
  1865. apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
  1866. apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
  1867. apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
  1868. apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
  1869. #if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
  1870. if (maxlvt >= 5)
  1871. apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
  1872. #endif
  1873. if (maxlvt >= 4)
  1874. apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
  1875. apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
  1876. apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
  1877. apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
  1878. apic_write(APIC_ESR, 0);
  1879. apic_read(APIC_ESR);
  1880. apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
  1881. apic_write(APIC_ESR, 0);
  1882. apic_read(APIC_ESR);
  1883. if (intr_remapping_enabled)
  1884. reenable_intr_remapping(x2apic_mode);
  1885. local_irq_restore(flags);
  1886. }
  1887. /*
  1888. * This device has no shutdown method - fully functioning local APICs
  1889. * are needed on every CPU up until machine_halt/restart/poweroff.
  1890. */
  1891. static struct syscore_ops lapic_syscore_ops = {
  1892. .resume = lapic_resume,
  1893. .suspend = lapic_suspend,
  1894. };
  1895. static void __cpuinit apic_pm_activate(void)
  1896. {
  1897. apic_pm_state.active = 1;
  1898. }
  1899. static int __init init_lapic_sysfs(void)
  1900. {
  1901. /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
  1902. if (cpu_has_apic)
  1903. register_syscore_ops(&lapic_syscore_ops);
  1904. return 0;
  1905. }
  1906. /* local apic needs to resume before other devices access its registers. */
  1907. core_initcall(init_lapic_sysfs);
  1908. #else /* CONFIG_PM */
  1909. static void apic_pm_activate(void) { }
  1910. #endif /* CONFIG_PM */
  1911. #ifdef CONFIG_X86_64
  1912. static int __cpuinit apic_cluster_num(void)
  1913. {
  1914. int i, clusters, zeros;
  1915. unsigned id;
  1916. u16 *bios_cpu_apicid;
  1917. DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);
  1918. bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
  1919. bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
  1920. for (i = 0; i < nr_cpu_ids; i++) {
  1921. /* are we being called early in kernel startup? */
  1922. if (bios_cpu_apicid) {
  1923. id = bios_cpu_apicid[i];
  1924. } else if (i < nr_cpu_ids) {
  1925. if (cpu_present(i))
  1926. id = per_cpu(x86_bios_cpu_apicid, i);
  1927. else
  1928. continue;
  1929. } else
  1930. break;
  1931. if (id != BAD_APICID)
  1932. __set_bit(APIC_CLUSTERID(id), clustermap);
  1933. }
  1934. /* Problem: Partially populated chassis may not have CPUs in some of
  1935. * the APIC clusters they have been allocated. Only present CPUs have
  1936. * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
  1937. * Since clusters are allocated sequentially, count zeros only if
  1938. * they are bounded by ones.
  1939. */
  1940. clusters = 0;
  1941. zeros = 0;
  1942. for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
  1943. if (test_bit(i, clustermap)) {
  1944. clusters += 1 + zeros;
  1945. zeros = 0;
  1946. } else
  1947. ++zeros;
  1948. }
  1949. return clusters;
  1950. }
  1951. static int __cpuinitdata multi_checked;
  1952. static int __cpuinitdata multi;
  1953. static int __cpuinit set_multi(const struct dmi_system_id *d)
  1954. {
  1955. if (multi)
  1956. return 0;
  1957. pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
  1958. multi = 1;
  1959. return 0;
  1960. }
  1961. static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = {
  1962. {
  1963. .callback = set_multi,
  1964. .ident = "IBM System Summit2",
  1965. .matches = {
  1966. DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
  1967. DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
  1968. },
  1969. },
  1970. {}
  1971. };
  1972. static void __cpuinit dmi_check_multi(void)
  1973. {
  1974. if (multi_checked)
  1975. return;
  1976. dmi_check_system(multi_dmi_table);
  1977. multi_checked = 1;
  1978. }
  1979. /*
  1980. * apic_is_clustered_box() -- Check if we can expect good TSC
  1981. *
  1982. * Thus far, the major user of this is IBM's Summit2 series:
  1983. * Clustered boxes may have unsynced TSC problems if they are
  1984. * multi-chassis.
  1985. * Use DMI to check them
  1986. */
  1987. __cpuinit int apic_is_clustered_box(void)
  1988. {
  1989. dmi_check_multi();
  1990. if (multi)
  1991. return 1;
  1992. if (!is_vsmp_box())
  1993. return 0;
  1994. /*
  1995. * ScaleMP vSMPowered boxes have one cluster per board and TSCs are
  1996. * not guaranteed to be synced between boards
  1997. */
  1998. if (apic_cluster_num() > 1)
  1999. return 1;
  2000. return 0;
  2001. }
  2002. #endif
  2003. /*
  2004. * APIC command line parameters
  2005. */
  2006. static int __init setup_disableapic(char *arg)
  2007. {
  2008. disable_apic = 1;
  2009. setup_clear_cpu_cap(X86_FEATURE_APIC);
  2010. return 0;
  2011. }
  2012. early_param("disableapic", setup_disableapic);
  2013. /* same as disableapic, for compatibility */
  2014. static int __init setup_nolapic(char *arg)
  2015. {
  2016. return setup_disableapic(arg);
  2017. }
  2018. early_param("nolapic", setup_nolapic);
  2019. static int __init parse_lapic_timer_c2_ok(char *arg)
  2020. {
  2021. local_apic_timer_c2_ok = 1;
  2022. return 0;
  2023. }
  2024. early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
  2025. static int __init parse_disable_apic_timer(char *arg)
  2026. {
  2027. disable_apic_timer = 1;
  2028. return 0;
  2029. }
  2030. early_param("noapictimer", parse_disable_apic_timer);
  2031. static int __init parse_nolapic_timer(char *arg)
  2032. {
  2033. disable_apic_timer = 1;
  2034. return 0;
  2035. }
  2036. early_param("nolapic_timer", parse_nolapic_timer);
  2037. static int __init apic_set_verbosity(char *arg)
  2038. {
  2039. if (!arg) {
  2040. #ifdef CONFIG_X86_64
  2041. skip_ioapic_setup = 0;
  2042. return 0;
  2043. #endif
  2044. return -EINVAL;
  2045. }
  2046. if (strcmp("debug", arg) == 0)
  2047. apic_verbosity = APIC_DEBUG;
  2048. else if (strcmp("verbose", arg) == 0)
  2049. apic_verbosity = APIC_VERBOSE;
  2050. else {
  2051. pr_warning("APIC Verbosity level %s not recognised"
  2052. " use apic=verbose or apic=debug\n", arg);
  2053. return -EINVAL;
  2054. }
  2055. return 0;
  2056. }
  2057. early_param("apic", apic_set_verbosity);
  2058. static int __init lapic_insert_resource(void)
  2059. {
  2060. if (!apic_phys)
  2061. return -1;
  2062. /* Put local APIC into the resource map. */
  2063. lapic_resource.start = apic_phys;
  2064. lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
  2065. insert_resource(&iomem_resource, &lapic_resource);
  2066. return 0;
  2067. }
  2068. /*
  2069. * need call insert after e820_reserve_resources()
  2070. * that is using request_resource
  2071. */
  2072. late_initcall(lapic_insert_resource);