apic.c 58 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415
  1. /*
  2. * Local APIC handling, local APIC timers
  3. *
  4. * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
  5. *
  6. * Fixes
  7. * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
  8. * thanks to Eric Gilmore
  9. * and Rolf G. Tews
  10. * for testing these extensively.
  11. * Maciej W. Rozycki : Various updates and fixes.
  12. * Mikael Pettersson : Power Management for UP-APIC.
  13. * Pavel Machek and
  14. * Mikael Pettersson : PM converted to driver model.
  15. */
  16. #include <linux/perf_event.h>
  17. #include <linux/kernel_stat.h>
  18. #include <linux/mc146818rtc.h>
  19. #include <linux/acpi_pmtmr.h>
  20. #include <linux/clockchips.h>
  21. #include <linux/interrupt.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/ftrace.h>
  24. #include <linux/ioport.h>
  25. #include <linux/module.h>
  26. #include <linux/syscore_ops.h>
  27. #include <linux/delay.h>
  28. #include <linux/timex.h>
  29. #include <linux/i8253.h>
  30. #include <linux/dmar.h>
  31. #include <linux/init.h>
  32. #include <linux/cpu.h>
  33. #include <linux/dmi.h>
  34. #include <linux/smp.h>
  35. #include <linux/mm.h>
  36. #include <asm/perf_event.h>
  37. #include <asm/x86_init.h>
  38. #include <asm/pgalloc.h>
  39. #include <linux/atomic.h>
  40. #include <asm/mpspec.h>
  41. #include <asm/i8259.h>
  42. #include <asm/proto.h>
  43. #include <asm/apic.h>
  44. #include <asm/io_apic.h>
  45. #include <asm/desc.h>
  46. #include <asm/hpet.h>
  47. #include <asm/idle.h>
  48. #include <asm/mtrr.h>
  49. #include <asm/time.h>
  50. #include <asm/smp.h>
  51. #include <asm/mce.h>
  52. #include <asm/tsc.h>
  53. #include <asm/hypervisor.h>
  54. unsigned int num_processors;
  55. unsigned disabled_cpus __cpuinitdata;
  56. /* Processor that is doing the boot up */
  57. unsigned int boot_cpu_physical_apicid = -1U;
  58. /*
  59. * The highest APIC ID seen during enumeration.
  60. */
  61. unsigned int max_physical_apicid;
  62. /*
  63. * Bitmask of physically existing CPUs:
  64. */
  65. physid_mask_t phys_cpu_present_map;
  66. /*
  67. * Map cpu index to physical APIC ID
  68. */
  69. DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
  70. DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
  71. EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
  72. EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  73. /*
  74. * ICR read retry counter
  75. */
  76. DEFINE_PER_CPU(unsigned, icr_read_retry_count);
  77. #ifdef CONFIG_X86_32
  78. /*
  79. * On x86_32, the mapping between cpu and logical apicid may vary
  80. * depending on apic in use. The following early percpu variable is
  81. * used for the mapping. This is where the behaviors of x86_64 and 32
  82. * actually diverge. Let's keep it ugly for now.
  83. */
  84. DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
  85. /*
  86. * Knob to control our willingness to enable the local APIC.
  87. *
  88. * +1=force-enable
  89. */
  90. static int force_enable_local_apic __initdata;
  91. /*
  92. * APIC command line parameters
  93. */
  94. static int __init parse_lapic(char *arg)
  95. {
  96. force_enable_local_apic = 1;
  97. return 0;
  98. }
  99. early_param("lapic", parse_lapic);
  100. /* Local APIC was disabled by the BIOS and enabled by the kernel */
  101. static int enabled_via_apicbase;
  102. /*
  103. * Handle interrupt mode configuration register (IMCR).
  104. * This register controls whether the interrupt signals
  105. * that reach the BSP come from the master PIC or from the
  106. * local APIC. Before entering Symmetric I/O Mode, either
  107. * the BIOS or the operating system must switch out of
  108. * PIC Mode by changing the IMCR.
  109. */
  110. static inline void imcr_pic_to_apic(void)
  111. {
  112. /* select IMCR register */
  113. outb(0x70, 0x22);
  114. /* NMI and 8259 INTR go through APIC */
  115. outb(0x01, 0x23);
  116. }
  117. static inline void imcr_apic_to_pic(void)
  118. {
  119. /* select IMCR register */
  120. outb(0x70, 0x22);
  121. /* NMI and 8259 INTR go directly to BSP */
  122. outb(0x00, 0x23);
  123. }
  124. #endif
  125. #ifdef CONFIG_X86_64
  126. static int apic_calibrate_pmtmr __initdata;
  127. static __init int setup_apicpmtimer(char *s)
  128. {
  129. apic_calibrate_pmtmr = 1;
  130. notsc_setup(NULL);
  131. return 0;
  132. }
  133. __setup("apicpmtimer", setup_apicpmtimer);
  134. #endif
  135. int x2apic_mode;
  136. #ifdef CONFIG_X86_X2APIC
  137. /* x2apic enabled before OS handover */
  138. static int x2apic_preenabled;
  139. static __init int setup_nox2apic(char *str)
  140. {
  141. if (x2apic_enabled()) {
  142. pr_warning("Bios already enabled x2apic, "
  143. "can't enforce nox2apic");
  144. return 0;
  145. }
  146. setup_clear_cpu_cap(X86_FEATURE_X2APIC);
  147. return 0;
  148. }
  149. early_param("nox2apic", setup_nox2apic);
  150. #endif
  151. unsigned long mp_lapic_addr;
  152. int disable_apic;
  153. /* Disable local APIC timer from the kernel commandline or via dmi quirk */
  154. static int disable_apic_timer __initdata;
  155. /* Local APIC timer works in C2 */
  156. int local_apic_timer_c2_ok;
  157. EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
  158. int first_system_vector = 0xfe;
  159. /*
  160. * Debug level, exported for io_apic.c
  161. */
  162. unsigned int apic_verbosity;
  163. int pic_mode;
  164. /* Have we found an MP table */
  165. int smp_found_config;
  166. static struct resource lapic_resource = {
  167. .name = "Local APIC",
  168. .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
  169. };
  170. unsigned int lapic_timer_frequency = 0;
  171. static void apic_pm_activate(void);
  172. static unsigned long apic_phys;
  173. /*
  174. * Get the LAPIC version
  175. */
  176. static inline int lapic_get_version(void)
  177. {
  178. return GET_APIC_VERSION(apic_read(APIC_LVR));
  179. }
  180. /*
  181. * Check, if the APIC is integrated or a separate chip
  182. */
  183. static inline int lapic_is_integrated(void)
  184. {
  185. #ifdef CONFIG_X86_64
  186. return 1;
  187. #else
  188. return APIC_INTEGRATED(lapic_get_version());
  189. #endif
  190. }
  191. /*
  192. * Check, whether this is a modern or a first generation APIC
  193. */
  194. static int modern_apic(void)
  195. {
  196. /* AMD systems use old APIC versions, so check the CPU */
  197. if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
  198. boot_cpu_data.x86 >= 0xf)
  199. return 1;
  200. return lapic_get_version() >= 0x14;
  201. }
  202. /*
  203. * right after this call apic become NOOP driven
  204. * so apic->write/read doesn't do anything
  205. */
  206. static void __init apic_disable(void)
  207. {
  208. pr_info("APIC: switched to apic NOOP\n");
  209. apic = &apic_noop;
  210. }
  211. void native_apic_wait_icr_idle(void)
  212. {
  213. while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
  214. cpu_relax();
  215. }
  216. u32 native_safe_apic_wait_icr_idle(void)
  217. {
  218. u32 send_status;
  219. int timeout;
  220. timeout = 0;
  221. do {
  222. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  223. if (!send_status)
  224. break;
  225. percpu_inc(icr_read_retry_count);
  226. udelay(100);
  227. } while (timeout++ < 1000);
  228. return send_status;
  229. }
  230. void native_apic_icr_write(u32 low, u32 id)
  231. {
  232. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
  233. apic_write(APIC_ICR, low);
  234. }
  235. u64 native_apic_icr_read(void)
  236. {
  237. u32 icr1, icr2;
  238. icr2 = apic_read(APIC_ICR2);
  239. icr1 = apic_read(APIC_ICR);
  240. return icr1 | ((u64)icr2 << 32);
  241. }
  242. #ifdef CONFIG_X86_32
  243. /**
  244. * get_physical_broadcast - Get number of physical broadcast IDs
  245. */
  246. int get_physical_broadcast(void)
  247. {
  248. return modern_apic() ? 0xff : 0xf;
  249. }
  250. #endif
  251. /**
  252. * lapic_get_maxlvt - get the maximum number of local vector table entries
  253. */
  254. int lapic_get_maxlvt(void)
  255. {
  256. unsigned int v;
  257. v = apic_read(APIC_LVR);
  258. /*
  259. * - we always have APIC integrated on 64bit mode
  260. * - 82489DXs do not report # of LVT entries
  261. */
  262. return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
  263. }
  264. /*
  265. * Local APIC timer
  266. */
  267. /* Clock divisor */
  268. #define APIC_DIVISOR 16
  269. /*
  270. * This function sets up the local APIC timer, with a timeout of
  271. * 'clocks' APIC bus clock. During calibration we actually call
  272. * this function twice on the boot CPU, once with a bogus timeout
  273. * value, second time for real. The other (noncalibrating) CPUs
  274. * call this function only once, with the real, calibrated value.
  275. *
  276. * We do reads before writes even if unnecessary, to get around the
  277. * P5 APIC double write bug.
  278. */
  279. static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
  280. {
  281. unsigned int lvtt_value, tmp_value;
  282. lvtt_value = LOCAL_TIMER_VECTOR;
  283. if (!oneshot)
  284. lvtt_value |= APIC_LVT_TIMER_PERIODIC;
  285. if (!lapic_is_integrated())
  286. lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
  287. if (!irqen)
  288. lvtt_value |= APIC_LVT_MASKED;
  289. apic_write(APIC_LVTT, lvtt_value);
  290. /*
  291. * Divide PICLK by 16
  292. */
  293. tmp_value = apic_read(APIC_TDCR);
  294. apic_write(APIC_TDCR,
  295. (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
  296. APIC_TDR_DIV_16);
  297. if (!oneshot)
  298. apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
  299. }
  300. /*
  301. * Setup extended LVT, AMD specific
  302. *
  303. * Software should use the LVT offsets the BIOS provides. The offsets
  304. * are determined by the subsystems using it like those for MCE
  305. * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
  306. * are supported. Beginning with family 10h at least 4 offsets are
  307. * available.
  308. *
  309. * Since the offsets must be consistent for all cores, we keep track
  310. * of the LVT offsets in software and reserve the offset for the same
  311. * vector also to be used on other cores. An offset is freed by
  312. * setting the entry to APIC_EILVT_MASKED.
  313. *
  314. * If the BIOS is right, there should be no conflicts. Otherwise a
  315. * "[Firmware Bug]: ..." error message is generated. However, if
  316. * software does not properly determines the offsets, it is not
  317. * necessarily a BIOS bug.
  318. */
  319. static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
  320. static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
  321. {
  322. return (old & APIC_EILVT_MASKED)
  323. || (new == APIC_EILVT_MASKED)
  324. || ((new & ~APIC_EILVT_MASKED) == old);
  325. }
  326. static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
  327. {
  328. unsigned int rsvd; /* 0: uninitialized */
  329. if (offset >= APIC_EILVT_NR_MAX)
  330. return ~0;
  331. rsvd = atomic_read(&eilvt_offsets[offset]) & ~APIC_EILVT_MASKED;
  332. do {
  333. if (rsvd &&
  334. !eilvt_entry_is_changeable(rsvd, new))
  335. /* may not change if vectors are different */
  336. return rsvd;
  337. rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
  338. } while (rsvd != new);
  339. return new;
  340. }
  341. /*
  342. * If mask=1, the LVT entry does not generate interrupts while mask=0
  343. * enables the vector. See also the BKDGs. Must be called with
  344. * preemption disabled.
  345. */
  346. int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
  347. {
  348. unsigned long reg = APIC_EILVTn(offset);
  349. unsigned int new, old, reserved;
  350. new = (mask << 16) | (msg_type << 8) | vector;
  351. old = apic_read(reg);
  352. reserved = reserve_eilvt_offset(offset, new);
  353. if (reserved != new) {
  354. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  355. "vector 0x%x, but the register is already in use for "
  356. "vector 0x%x on another cpu\n",
  357. smp_processor_id(), reg, offset, new, reserved);
  358. return -EINVAL;
  359. }
  360. if (!eilvt_entry_is_changeable(old, new)) {
  361. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  362. "vector 0x%x, but the register is already in use for "
  363. "vector 0x%x on this cpu\n",
  364. smp_processor_id(), reg, offset, new, old);
  365. return -EBUSY;
  366. }
  367. apic_write(reg, new);
  368. return 0;
  369. }
  370. EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
  371. /*
  372. * Program the next event, relative to now
  373. */
  374. static int lapic_next_event(unsigned long delta,
  375. struct clock_event_device *evt)
  376. {
  377. apic_write(APIC_TMICT, delta);
  378. return 0;
  379. }
  380. /*
  381. * Setup the lapic timer in periodic or oneshot mode
  382. */
  383. static void lapic_timer_setup(enum clock_event_mode mode,
  384. struct clock_event_device *evt)
  385. {
  386. unsigned long flags;
  387. unsigned int v;
  388. /* Lapic used as dummy for broadcast ? */
  389. if (evt->features & CLOCK_EVT_FEAT_DUMMY)
  390. return;
  391. local_irq_save(flags);
  392. switch (mode) {
  393. case CLOCK_EVT_MODE_PERIODIC:
  394. case CLOCK_EVT_MODE_ONESHOT:
  395. __setup_APIC_LVTT(lapic_timer_frequency,
  396. mode != CLOCK_EVT_MODE_PERIODIC, 1);
  397. break;
  398. case CLOCK_EVT_MODE_UNUSED:
  399. case CLOCK_EVT_MODE_SHUTDOWN:
  400. v = apic_read(APIC_LVTT);
  401. v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  402. apic_write(APIC_LVTT, v);
  403. apic_write(APIC_TMICT, 0);
  404. break;
  405. case CLOCK_EVT_MODE_RESUME:
  406. /* Nothing to do here */
  407. break;
  408. }
  409. local_irq_restore(flags);
  410. }
  411. /*
  412. * Local APIC timer broadcast function
  413. */
  414. static void lapic_timer_broadcast(const struct cpumask *mask)
  415. {
  416. #ifdef CONFIG_SMP
  417. apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
  418. #endif
  419. }
  420. /*
  421. * The local apic timer can be used for any function which is CPU local.
  422. */
  423. static struct clock_event_device lapic_clockevent = {
  424. .name = "lapic",
  425. .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
  426. | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
  427. .shift = 32,
  428. .set_mode = lapic_timer_setup,
  429. .set_next_event = lapic_next_event,
  430. .broadcast = lapic_timer_broadcast,
  431. .rating = 100,
  432. .irq = -1,
  433. };
  434. static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
  435. /*
  436. * Setup the local APIC timer for this CPU. Copy the initialized values
  437. * of the boot CPU and register the clock event in the framework.
  438. */
  439. static void __cpuinit setup_APIC_timer(void)
  440. {
  441. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  442. if (this_cpu_has(X86_FEATURE_ARAT)) {
  443. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
  444. /* Make LAPIC timer preferrable over percpu HPET */
  445. lapic_clockevent.rating = 150;
  446. }
  447. memcpy(levt, &lapic_clockevent, sizeof(*levt));
  448. levt->cpumask = cpumask_of(smp_processor_id());
  449. clockevents_register_device(levt);
  450. }
  451. /*
  452. * In this functions we calibrate APIC bus clocks to the external timer.
  453. *
  454. * We want to do the calibration only once since we want to have local timer
  455. * irqs syncron. CPUs connected by the same APIC bus have the very same bus
  456. * frequency.
  457. *
  458. * This was previously done by reading the PIT/HPET and waiting for a wrap
  459. * around to find out, that a tick has elapsed. I have a box, where the PIT
  460. * readout is broken, so it never gets out of the wait loop again. This was
  461. * also reported by others.
  462. *
  463. * Monitoring the jiffies value is inaccurate and the clockevents
  464. * infrastructure allows us to do a simple substitution of the interrupt
  465. * handler.
  466. *
  467. * The calibration routine also uses the pm_timer when possible, as the PIT
  468. * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
  469. * back to normal later in the boot process).
  470. */
  471. #define LAPIC_CAL_LOOPS (HZ/10)
  472. static __initdata int lapic_cal_loops = -1;
  473. static __initdata long lapic_cal_t1, lapic_cal_t2;
  474. static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
  475. static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
  476. static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
  477. /*
  478. * Temporary interrupt handler.
  479. */
  480. static void __init lapic_cal_handler(struct clock_event_device *dev)
  481. {
  482. unsigned long long tsc = 0;
  483. long tapic = apic_read(APIC_TMCCT);
  484. unsigned long pm = acpi_pm_read_early();
  485. if (cpu_has_tsc)
  486. rdtscll(tsc);
  487. switch (lapic_cal_loops++) {
  488. case 0:
  489. lapic_cal_t1 = tapic;
  490. lapic_cal_tsc1 = tsc;
  491. lapic_cal_pm1 = pm;
  492. lapic_cal_j1 = jiffies;
  493. break;
  494. case LAPIC_CAL_LOOPS:
  495. lapic_cal_t2 = tapic;
  496. lapic_cal_tsc2 = tsc;
  497. if (pm < lapic_cal_pm1)
  498. pm += ACPI_PM_OVRRUN;
  499. lapic_cal_pm2 = pm;
  500. lapic_cal_j2 = jiffies;
  501. break;
  502. }
  503. }
  504. static int __init
  505. calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
  506. {
  507. const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
  508. const long pm_thresh = pm_100ms / 100;
  509. unsigned long mult;
  510. u64 res;
  511. #ifndef CONFIG_X86_PM_TIMER
  512. return -1;
  513. #endif
  514. apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
  515. /* Check, if the PM timer is available */
  516. if (!deltapm)
  517. return -1;
  518. mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
  519. if (deltapm > (pm_100ms - pm_thresh) &&
  520. deltapm < (pm_100ms + pm_thresh)) {
  521. apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
  522. return 0;
  523. }
  524. res = (((u64)deltapm) * mult) >> 22;
  525. do_div(res, 1000000);
  526. pr_warning("APIC calibration not consistent "
  527. "with PM-Timer: %ldms instead of 100ms\n",(long)res);
  528. /* Correct the lapic counter value */
  529. res = (((u64)(*delta)) * pm_100ms);
  530. do_div(res, deltapm);
  531. pr_info("APIC delta adjusted to PM-Timer: "
  532. "%lu (%ld)\n", (unsigned long)res, *delta);
  533. *delta = (long)res;
  534. /* Correct the tsc counter value */
  535. if (cpu_has_tsc) {
  536. res = (((u64)(*deltatsc)) * pm_100ms);
  537. do_div(res, deltapm);
  538. apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
  539. "PM-Timer: %lu (%ld)\n",
  540. (unsigned long)res, *deltatsc);
  541. *deltatsc = (long)res;
  542. }
  543. return 0;
  544. }
  545. static int __init calibrate_APIC_clock(void)
  546. {
  547. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  548. void (*real_handler)(struct clock_event_device *dev);
  549. unsigned long deltaj;
  550. long delta, deltatsc;
  551. int pm_referenced = 0;
  552. /**
  553. * check if lapic timer has already been calibrated by platform
  554. * specific routine, such as tsc calibration code. if so, we just fill
  555. * in the clockevent structure and return.
  556. */
  557. if (lapic_timer_frequency) {
  558. apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
  559. lapic_timer_frequency);
  560. lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
  561. TICK_NSEC, lapic_clockevent.shift);
  562. lapic_clockevent.max_delta_ns =
  563. clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
  564. lapic_clockevent.min_delta_ns =
  565. clockevent_delta2ns(0xF, &lapic_clockevent);
  566. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
  567. return 0;
  568. }
  569. local_irq_disable();
  570. /* Replace the global interrupt handler */
  571. real_handler = global_clock_event->event_handler;
  572. global_clock_event->event_handler = lapic_cal_handler;
  573. /*
  574. * Setup the APIC counter to maximum. There is no way the lapic
  575. * can underflow in the 100ms detection time frame
  576. */
  577. __setup_APIC_LVTT(0xffffffff, 0, 0);
  578. /* Let the interrupts run */
  579. local_irq_enable();
  580. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  581. cpu_relax();
  582. local_irq_disable();
  583. /* Restore the real event handler */
  584. global_clock_event->event_handler = real_handler;
  585. /* Build delta t1-t2 as apic timer counts down */
  586. delta = lapic_cal_t1 - lapic_cal_t2;
  587. apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
  588. deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
  589. /* we trust the PM based calibration if possible */
  590. pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
  591. &delta, &deltatsc);
  592. /* Calculate the scaled math multiplication factor */
  593. lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
  594. lapic_clockevent.shift);
  595. lapic_clockevent.max_delta_ns =
  596. clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
  597. lapic_clockevent.min_delta_ns =
  598. clockevent_delta2ns(0xF, &lapic_clockevent);
  599. lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
  600. apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
  601. apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
  602. apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
  603. lapic_timer_frequency);
  604. if (cpu_has_tsc) {
  605. apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
  606. "%ld.%04ld MHz.\n",
  607. (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
  608. (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
  609. }
  610. apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
  611. "%u.%04u MHz.\n",
  612. lapic_timer_frequency / (1000000 / HZ),
  613. lapic_timer_frequency % (1000000 / HZ));
  614. /*
  615. * Do a sanity check on the APIC calibration result
  616. */
  617. if (lapic_timer_frequency < (1000000 / HZ)) {
  618. local_irq_enable();
  619. pr_warning("APIC frequency too slow, disabling apic timer\n");
  620. return -1;
  621. }
  622. levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
  623. /*
  624. * PM timer calibration failed or not turned on
  625. * so lets try APIC timer based calibration
  626. */
  627. if (!pm_referenced) {
  628. apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
  629. /*
  630. * Setup the apic timer manually
  631. */
  632. levt->event_handler = lapic_cal_handler;
  633. lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
  634. lapic_cal_loops = -1;
  635. /* Let the interrupts run */
  636. local_irq_enable();
  637. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  638. cpu_relax();
  639. /* Stop the lapic timer */
  640. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
  641. /* Jiffies delta */
  642. deltaj = lapic_cal_j2 - lapic_cal_j1;
  643. apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
  644. /* Check, if the jiffies result is consistent */
  645. if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
  646. apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
  647. else
  648. levt->features |= CLOCK_EVT_FEAT_DUMMY;
  649. } else
  650. local_irq_enable();
  651. if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
  652. pr_warning("APIC timer disabled due to verification failure\n");
  653. return -1;
  654. }
  655. return 0;
  656. }
  657. /*
  658. * Setup the boot APIC
  659. *
  660. * Calibrate and verify the result.
  661. */
  662. void __init setup_boot_APIC_clock(void)
  663. {
  664. /*
  665. * The local apic timer can be disabled via the kernel
  666. * commandline or from the CPU detection code. Register the lapic
  667. * timer as a dummy clock event source on SMP systems, so the
  668. * broadcast mechanism is used. On UP systems simply ignore it.
  669. */
  670. if (disable_apic_timer) {
  671. pr_info("Disabling APIC timer\n");
  672. /* No broadcast on UP ! */
  673. if (num_possible_cpus() > 1) {
  674. lapic_clockevent.mult = 1;
  675. setup_APIC_timer();
  676. }
  677. return;
  678. }
  679. apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
  680. "calibrating APIC timer ...\n");
  681. if (calibrate_APIC_clock()) {
  682. /* No broadcast on UP ! */
  683. if (num_possible_cpus() > 1)
  684. setup_APIC_timer();
  685. return;
  686. }
  687. /*
  688. * If nmi_watchdog is set to IO_APIC, we need the
  689. * PIT/HPET going. Otherwise register lapic as a dummy
  690. * device.
  691. */
  692. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
  693. /* Setup the lapic or request the broadcast */
  694. setup_APIC_timer();
  695. }
  696. void __cpuinit setup_secondary_APIC_clock(void)
  697. {
  698. setup_APIC_timer();
  699. }
  700. /*
  701. * The guts of the apic timer interrupt
  702. */
  703. static void local_apic_timer_interrupt(void)
  704. {
  705. int cpu = smp_processor_id();
  706. struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
  707. /*
  708. * Normally we should not be here till LAPIC has been initialized but
  709. * in some cases like kdump, its possible that there is a pending LAPIC
  710. * timer interrupt from previous kernel's context and is delivered in
  711. * new kernel the moment interrupts are enabled.
  712. *
  713. * Interrupts are enabled early and LAPIC is setup much later, hence
  714. * its possible that when we get here evt->event_handler is NULL.
  715. * Check for event_handler being NULL and discard the interrupt as
  716. * spurious.
  717. */
  718. if (!evt->event_handler) {
  719. pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
  720. /* Switch it off */
  721. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
  722. return;
  723. }
  724. /*
  725. * the NMI deadlock-detector uses this.
  726. */
  727. inc_irq_stat(apic_timer_irqs);
  728. evt->event_handler(evt);
  729. }
  730. /*
  731. * Local APIC timer interrupt. This is the most natural way for doing
  732. * local interrupts, but local timer interrupts can be emulated by
  733. * broadcast interrupts too. [in case the hw doesn't support APIC timers]
  734. *
  735. * [ if a single-CPU system runs an SMP kernel then we call the local
  736. * interrupt as well. Thus we cannot inline the local irq ... ]
  737. */
  738. void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
  739. {
  740. struct pt_regs *old_regs = set_irq_regs(regs);
  741. /*
  742. * NOTE! We'd better ACK the irq immediately,
  743. * because timer handling can be slow.
  744. */
  745. ack_APIC_irq();
  746. /*
  747. * update_process_times() expects us to have done irq_enter().
  748. * Besides, if we don't timer interrupts ignore the global
  749. * interrupt lock, which is the WrongThing (tm) to do.
  750. */
  751. exit_idle();
  752. irq_enter();
  753. local_apic_timer_interrupt();
  754. irq_exit();
  755. set_irq_regs(old_regs);
  756. }
  757. int setup_profiling_timer(unsigned int multiplier)
  758. {
  759. return -EINVAL;
  760. }
  761. /*
  762. * Local APIC start and shutdown
  763. */
  764. /**
  765. * clear_local_APIC - shutdown the local APIC
  766. *
  767. * This is called, when a CPU is disabled and before rebooting, so the state of
  768. * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
  769. * leftovers during boot.
  770. */
  771. void clear_local_APIC(void)
  772. {
  773. int maxlvt;
  774. u32 v;
  775. /* APIC hasn't been mapped yet */
  776. if (!x2apic_mode && !apic_phys)
  777. return;
  778. maxlvt = lapic_get_maxlvt();
  779. /*
  780. * Masking an LVT entry can trigger a local APIC error
  781. * if the vector is zero. Mask LVTERR first to prevent this.
  782. */
  783. if (maxlvt >= 3) {
  784. v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
  785. apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
  786. }
  787. /*
  788. * Careful: we have to set masks only first to deassert
  789. * any level-triggered sources.
  790. */
  791. v = apic_read(APIC_LVTT);
  792. apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
  793. v = apic_read(APIC_LVT0);
  794. apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
  795. v = apic_read(APIC_LVT1);
  796. apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
  797. if (maxlvt >= 4) {
  798. v = apic_read(APIC_LVTPC);
  799. apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
  800. }
  801. /* lets not touch this if we didn't frob it */
  802. #ifdef CONFIG_X86_THERMAL_VECTOR
  803. if (maxlvt >= 5) {
  804. v = apic_read(APIC_LVTTHMR);
  805. apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
  806. }
  807. #endif
  808. #ifdef CONFIG_X86_MCE_INTEL
  809. if (maxlvt >= 6) {
  810. v = apic_read(APIC_LVTCMCI);
  811. if (!(v & APIC_LVT_MASKED))
  812. apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
  813. }
  814. #endif
  815. /*
  816. * Clean APIC state for other OSs:
  817. */
  818. apic_write(APIC_LVTT, APIC_LVT_MASKED);
  819. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  820. apic_write(APIC_LVT1, APIC_LVT_MASKED);
  821. if (maxlvt >= 3)
  822. apic_write(APIC_LVTERR, APIC_LVT_MASKED);
  823. if (maxlvt >= 4)
  824. apic_write(APIC_LVTPC, APIC_LVT_MASKED);
  825. /* Integrated APIC (!82489DX) ? */
  826. if (lapic_is_integrated()) {
  827. if (maxlvt > 3)
  828. /* Clear ESR due to Pentium errata 3AP and 11AP */
  829. apic_write(APIC_ESR, 0);
  830. apic_read(APIC_ESR);
  831. }
  832. }
  833. /**
  834. * disable_local_APIC - clear and disable the local APIC
  835. */
  836. void disable_local_APIC(void)
  837. {
  838. unsigned int value;
  839. /* APIC hasn't been mapped yet */
  840. if (!x2apic_mode && !apic_phys)
  841. return;
  842. clear_local_APIC();
  843. /*
  844. * Disable APIC (implies clearing of registers
  845. * for 82489DX!).
  846. */
  847. value = apic_read(APIC_SPIV);
  848. value &= ~APIC_SPIV_APIC_ENABLED;
  849. apic_write(APIC_SPIV, value);
  850. #ifdef CONFIG_X86_32
  851. /*
  852. * When LAPIC was disabled by the BIOS and enabled by the kernel,
  853. * restore the disabled state.
  854. */
  855. if (enabled_via_apicbase) {
  856. unsigned int l, h;
  857. rdmsr(MSR_IA32_APICBASE, l, h);
  858. l &= ~MSR_IA32_APICBASE_ENABLE;
  859. wrmsr(MSR_IA32_APICBASE, l, h);
  860. }
  861. #endif
  862. }
  863. /*
  864. * If Linux enabled the LAPIC against the BIOS default disable it down before
  865. * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
  866. * not power-off. Additionally clear all LVT entries before disable_local_APIC
  867. * for the case where Linux didn't enable the LAPIC.
  868. */
  869. void lapic_shutdown(void)
  870. {
  871. unsigned long flags;
  872. if (!cpu_has_apic && !apic_from_smp_config())
  873. return;
  874. local_irq_save(flags);
  875. #ifdef CONFIG_X86_32
  876. if (!enabled_via_apicbase)
  877. clear_local_APIC();
  878. else
  879. #endif
  880. disable_local_APIC();
  881. local_irq_restore(flags);
  882. }
  883. /*
  884. * This is to verify that we're looking at a real local APIC.
  885. * Check these against your board if the CPUs aren't getting
  886. * started for no apparent reason.
  887. */
  888. int __init verify_local_APIC(void)
  889. {
  890. unsigned int reg0, reg1;
  891. /*
  892. * The version register is read-only in a real APIC.
  893. */
  894. reg0 = apic_read(APIC_LVR);
  895. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
  896. apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
  897. reg1 = apic_read(APIC_LVR);
  898. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
  899. /*
  900. * The two version reads above should print the same
  901. * numbers. If the second one is different, then we
  902. * poke at a non-APIC.
  903. */
  904. if (reg1 != reg0)
  905. return 0;
  906. /*
  907. * Check if the version looks reasonably.
  908. */
  909. reg1 = GET_APIC_VERSION(reg0);
  910. if (reg1 == 0x00 || reg1 == 0xff)
  911. return 0;
  912. reg1 = lapic_get_maxlvt();
  913. if (reg1 < 0x02 || reg1 == 0xff)
  914. return 0;
  915. /*
  916. * The ID register is read/write in a real APIC.
  917. */
  918. reg0 = apic_read(APIC_ID);
  919. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
  920. apic_write(APIC_ID, reg0 ^ apic->apic_id_mask);
  921. reg1 = apic_read(APIC_ID);
  922. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
  923. apic_write(APIC_ID, reg0);
  924. if (reg1 != (reg0 ^ apic->apic_id_mask))
  925. return 0;
  926. /*
  927. * The next two are just to see if we have sane values.
  928. * They're only really relevant if we're in Virtual Wire
  929. * compatibility mode, but most boxes are anymore.
  930. */
  931. reg0 = apic_read(APIC_LVT0);
  932. apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
  933. reg1 = apic_read(APIC_LVT1);
  934. apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
  935. return 1;
  936. }
  937. /**
  938. * sync_Arb_IDs - synchronize APIC bus arbitration IDs
  939. */
  940. void __init sync_Arb_IDs(void)
  941. {
  942. /*
  943. * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
  944. * needed on AMD.
  945. */
  946. if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
  947. return;
  948. /*
  949. * Wait for idle.
  950. */
  951. apic_wait_icr_idle();
  952. apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
  953. apic_write(APIC_ICR, APIC_DEST_ALLINC |
  954. APIC_INT_LEVELTRIG | APIC_DM_INIT);
  955. }
  956. /*
  957. * An initial setup of the virtual wire mode.
  958. */
  959. void __init init_bsp_APIC(void)
  960. {
  961. unsigned int value;
  962. /*
  963. * Don't do the setup now if we have a SMP BIOS as the
  964. * through-I/O-APIC virtual wire mode might be active.
  965. */
  966. if (smp_found_config || !cpu_has_apic)
  967. return;
  968. /*
  969. * Do not trust the local APIC being empty at bootup.
  970. */
  971. clear_local_APIC();
  972. /*
  973. * Enable APIC.
  974. */
  975. value = apic_read(APIC_SPIV);
  976. value &= ~APIC_VECTOR_MASK;
  977. value |= APIC_SPIV_APIC_ENABLED;
  978. #ifdef CONFIG_X86_32
  979. /* This bit is reserved on P4/Xeon and should be cleared */
  980. if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
  981. (boot_cpu_data.x86 == 15))
  982. value &= ~APIC_SPIV_FOCUS_DISABLED;
  983. else
  984. #endif
  985. value |= APIC_SPIV_FOCUS_DISABLED;
  986. value |= SPURIOUS_APIC_VECTOR;
  987. apic_write(APIC_SPIV, value);
  988. /*
  989. * Set up the virtual wire mode.
  990. */
  991. apic_write(APIC_LVT0, APIC_DM_EXTINT);
  992. value = APIC_DM_NMI;
  993. if (!lapic_is_integrated()) /* 82489DX */
  994. value |= APIC_LVT_LEVEL_TRIGGER;
  995. apic_write(APIC_LVT1, value);
  996. }
  997. static void __cpuinit lapic_setup_esr(void)
  998. {
  999. unsigned int oldvalue, value, maxlvt;
  1000. if (!lapic_is_integrated()) {
  1001. pr_info("No ESR for 82489DX.\n");
  1002. return;
  1003. }
  1004. if (apic->disable_esr) {
  1005. /*
  1006. * Something untraceable is creating bad interrupts on
  1007. * secondary quads ... for the moment, just leave the
  1008. * ESR disabled - we can't do anything useful with the
  1009. * errors anyway - mbligh
  1010. */
  1011. pr_info("Leaving ESR disabled.\n");
  1012. return;
  1013. }
  1014. maxlvt = lapic_get_maxlvt();
  1015. if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
  1016. apic_write(APIC_ESR, 0);
  1017. oldvalue = apic_read(APIC_ESR);
  1018. /* enables sending errors */
  1019. value = ERROR_APIC_VECTOR;
  1020. apic_write(APIC_LVTERR, value);
  1021. /*
  1022. * spec says clear errors after enabling vector.
  1023. */
  1024. if (maxlvt > 3)
  1025. apic_write(APIC_ESR, 0);
  1026. value = apic_read(APIC_ESR);
  1027. if (value != oldvalue)
  1028. apic_printk(APIC_VERBOSE, "ESR value before enabling "
  1029. "vector: 0x%08x after: 0x%08x\n",
  1030. oldvalue, value);
  1031. }
  1032. /**
  1033. * setup_local_APIC - setup the local APIC
  1034. *
  1035. * Used to setup local APIC while initializing BSP or bringin up APs.
  1036. * Always called with preemption disabled.
  1037. */
  1038. void __cpuinit setup_local_APIC(void)
  1039. {
  1040. int cpu = smp_processor_id();
  1041. unsigned int value, queued;
  1042. int i, j, acked = 0;
  1043. unsigned long long tsc = 0, ntsc;
  1044. long long max_loops = cpu_khz;
  1045. if (cpu_has_tsc)
  1046. rdtscll(tsc);
  1047. if (disable_apic) {
  1048. disable_ioapic_support();
  1049. return;
  1050. }
  1051. #ifdef CONFIG_X86_32
  1052. /* Pound the ESR really hard over the head with a big hammer - mbligh */
  1053. if (lapic_is_integrated() && apic->disable_esr) {
  1054. apic_write(APIC_ESR, 0);
  1055. apic_write(APIC_ESR, 0);
  1056. apic_write(APIC_ESR, 0);
  1057. apic_write(APIC_ESR, 0);
  1058. }
  1059. #endif
  1060. perf_events_lapic_init();
  1061. /*
  1062. * Double-check whether this APIC is really registered.
  1063. * This is meaningless in clustered apic mode, so we skip it.
  1064. */
  1065. BUG_ON(!apic->apic_id_registered());
  1066. /*
  1067. * Intel recommends to set DFR, LDR and TPR before enabling
  1068. * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
  1069. * document number 292116). So here it goes...
  1070. */
  1071. apic->init_apic_ldr();
  1072. #ifdef CONFIG_X86_32
  1073. /*
  1074. * APIC LDR is initialized. If logical_apicid mapping was
  1075. * initialized during get_smp_config(), make sure it matches the
  1076. * actual value.
  1077. */
  1078. i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
  1079. WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
  1080. /* always use the value from LDR */
  1081. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1082. logical_smp_processor_id();
  1083. /*
  1084. * Some NUMA implementations (NUMAQ) don't initialize apicid to
  1085. * node mapping during NUMA init. Now that logical apicid is
  1086. * guaranteed to be known, give it another chance. This is already
  1087. * a bit too late - percpu allocation has already happened without
  1088. * proper NUMA affinity.
  1089. */
  1090. if (apic->x86_32_numa_cpu_node)
  1091. set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu),
  1092. apic->x86_32_numa_cpu_node(cpu));
  1093. #endif
  1094. /*
  1095. * Set Task Priority to 'accept all'. We never change this
  1096. * later on.
  1097. */
  1098. value = apic_read(APIC_TASKPRI);
  1099. value &= ~APIC_TPRI_MASK;
  1100. apic_write(APIC_TASKPRI, value);
  1101. /*
  1102. * After a crash, we no longer service the interrupts and a pending
  1103. * interrupt from previous kernel might still have ISR bit set.
  1104. *
  1105. * Most probably by now CPU has serviced that pending interrupt and
  1106. * it might not have done the ack_APIC_irq() because it thought,
  1107. * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
  1108. * does not clear the ISR bit and cpu thinks it has already serivced
  1109. * the interrupt. Hence a vector might get locked. It was noticed
  1110. * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
  1111. */
  1112. do {
  1113. queued = 0;
  1114. for (i = APIC_ISR_NR - 1; i >= 0; i--)
  1115. queued |= apic_read(APIC_IRR + i*0x10);
  1116. for (i = APIC_ISR_NR - 1; i >= 0; i--) {
  1117. value = apic_read(APIC_ISR + i*0x10);
  1118. for (j = 31; j >= 0; j--) {
  1119. if (value & (1<<j)) {
  1120. ack_APIC_irq();
  1121. acked++;
  1122. }
  1123. }
  1124. }
  1125. if (acked > 256) {
  1126. printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
  1127. acked);
  1128. break;
  1129. }
  1130. if (cpu_has_tsc) {
  1131. rdtscll(ntsc);
  1132. max_loops = (cpu_khz << 10) - (ntsc - tsc);
  1133. } else
  1134. max_loops--;
  1135. } while (queued && max_loops > 0);
  1136. WARN_ON(max_loops <= 0);
  1137. /*
  1138. * Now that we are all set up, enable the APIC
  1139. */
  1140. value = apic_read(APIC_SPIV);
  1141. value &= ~APIC_VECTOR_MASK;
  1142. /*
  1143. * Enable APIC
  1144. */
  1145. value |= APIC_SPIV_APIC_ENABLED;
  1146. #ifdef CONFIG_X86_32
  1147. /*
  1148. * Some unknown Intel IO/APIC (or APIC) errata is biting us with
  1149. * certain networking cards. If high frequency interrupts are
  1150. * happening on a particular IOAPIC pin, plus the IOAPIC routing
  1151. * entry is masked/unmasked at a high rate as well then sooner or
  1152. * later IOAPIC line gets 'stuck', no more interrupts are received
  1153. * from the device. If focus CPU is disabled then the hang goes
  1154. * away, oh well :-(
  1155. *
  1156. * [ This bug can be reproduced easily with a level-triggered
  1157. * PCI Ne2000 networking cards and PII/PIII processors, dual
  1158. * BX chipset. ]
  1159. */
  1160. /*
  1161. * Actually disabling the focus CPU check just makes the hang less
  1162. * frequent as it makes the interrupt distributon model be more
  1163. * like LRU than MRU (the short-term load is more even across CPUs).
  1164. * See also the comment in end_level_ioapic_irq(). --macro
  1165. */
  1166. /*
  1167. * - enable focus processor (bit==0)
  1168. * - 64bit mode always use processor focus
  1169. * so no need to set it
  1170. */
  1171. value &= ~APIC_SPIV_FOCUS_DISABLED;
  1172. #endif
  1173. /*
  1174. * Set spurious IRQ vector
  1175. */
  1176. value |= SPURIOUS_APIC_VECTOR;
  1177. apic_write(APIC_SPIV, value);
  1178. /*
  1179. * Set up LVT0, LVT1:
  1180. *
  1181. * set up through-local-APIC on the BP's LINT0. This is not
  1182. * strictly necessary in pure symmetric-IO mode, but sometimes
  1183. * we delegate interrupts to the 8259A.
  1184. */
  1185. /*
  1186. * TODO: set up through-local-APIC from through-I/O-APIC? --macro
  1187. */
  1188. value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
  1189. if (!cpu && (pic_mode || !value)) {
  1190. value = APIC_DM_EXTINT;
  1191. apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
  1192. } else {
  1193. value = APIC_DM_EXTINT | APIC_LVT_MASKED;
  1194. apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
  1195. }
  1196. apic_write(APIC_LVT0, value);
  1197. /*
  1198. * only the BP should see the LINT1 NMI signal, obviously.
  1199. */
  1200. if (!cpu)
  1201. value = APIC_DM_NMI;
  1202. else
  1203. value = APIC_DM_NMI | APIC_LVT_MASKED;
  1204. if (!lapic_is_integrated()) /* 82489DX */
  1205. value |= APIC_LVT_LEVEL_TRIGGER;
  1206. apic_write(APIC_LVT1, value);
  1207. #ifdef CONFIG_X86_MCE_INTEL
  1208. /* Recheck CMCI information after local APIC is up on CPU #0 */
  1209. if (!cpu)
  1210. cmci_recheck();
  1211. #endif
  1212. }
  1213. void __cpuinit end_local_APIC_setup(void)
  1214. {
  1215. lapic_setup_esr();
  1216. #ifdef CONFIG_X86_32
  1217. {
  1218. unsigned int value;
  1219. /* Disable the local apic timer */
  1220. value = apic_read(APIC_LVTT);
  1221. value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  1222. apic_write(APIC_LVTT, value);
  1223. }
  1224. #endif
  1225. apic_pm_activate();
  1226. }
  1227. void __init bsp_end_local_APIC_setup(void)
  1228. {
  1229. end_local_APIC_setup();
  1230. /*
  1231. * Now that local APIC setup is completed for BP, configure the fault
  1232. * handling for interrupt remapping.
  1233. */
  1234. if (intr_remapping_enabled)
  1235. enable_drhd_fault_handling();
  1236. }
  1237. #ifdef CONFIG_X86_X2APIC
  1238. void check_x2apic(void)
  1239. {
  1240. if (x2apic_enabled()) {
  1241. pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
  1242. x2apic_preenabled = x2apic_mode = 1;
  1243. }
  1244. }
  1245. void enable_x2apic(void)
  1246. {
  1247. int msr, msr2;
  1248. if (!x2apic_mode)
  1249. return;
  1250. rdmsr(MSR_IA32_APICBASE, msr, msr2);
  1251. if (!(msr & X2APIC_ENABLE)) {
  1252. printk_once(KERN_INFO "Enabling x2apic\n");
  1253. wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2);
  1254. }
  1255. }
  1256. #endif /* CONFIG_X86_X2APIC */
  1257. int __init enable_IR(void)
  1258. {
  1259. #ifdef CONFIG_IRQ_REMAP
  1260. if (!intr_remapping_supported()) {
  1261. pr_debug("intr-remapping not supported\n");
  1262. return -1;
  1263. }
  1264. if (!x2apic_preenabled && skip_ioapic_setup) {
  1265. pr_info("Skipped enabling intr-remap because of skipping "
  1266. "io-apic setup\n");
  1267. return -1;
  1268. }
  1269. return enable_intr_remapping();
  1270. #endif
  1271. return -1;
  1272. }
  1273. void __init enable_IR_x2apic(void)
  1274. {
  1275. unsigned long flags;
  1276. int ret, x2apic_enabled = 0;
  1277. int dmar_table_init_ret;
  1278. dmar_table_init_ret = dmar_table_init();
  1279. if (dmar_table_init_ret && !x2apic_supported())
  1280. return;
  1281. ret = save_ioapic_entries();
  1282. if (ret) {
  1283. pr_info("Saving IO-APIC state failed: %d\n", ret);
  1284. goto out;
  1285. }
  1286. local_irq_save(flags);
  1287. legacy_pic->mask_all();
  1288. mask_ioapic_entries();
  1289. if (dmar_table_init_ret)
  1290. ret = -1;
  1291. else
  1292. ret = enable_IR();
  1293. if (ret < 0) {
  1294. /* IR is required if there is APIC ID > 255 even when running
  1295. * under KVM
  1296. */
  1297. if (max_physical_apicid > 255 ||
  1298. !hypervisor_x2apic_available())
  1299. goto nox2apic;
  1300. /*
  1301. * without IR all CPUs can be addressed by IOAPIC/MSI
  1302. * only in physical mode
  1303. */
  1304. x2apic_force_phys();
  1305. }
  1306. if (ret == IRQ_REMAP_XAPIC_MODE)
  1307. goto nox2apic;
  1308. x2apic_enabled = 1;
  1309. if (x2apic_supported() && !x2apic_mode) {
  1310. x2apic_mode = 1;
  1311. enable_x2apic();
  1312. pr_info("Enabled x2apic\n");
  1313. }
  1314. nox2apic:
  1315. if (ret < 0) /* IR enabling failed */
  1316. restore_ioapic_entries();
  1317. legacy_pic->restore_mask();
  1318. local_irq_restore(flags);
  1319. out:
  1320. if (x2apic_enabled || !x2apic_supported())
  1321. return;
  1322. if (x2apic_preenabled)
  1323. panic("x2apic: enabled by BIOS but kernel init failed.");
  1324. else if (ret == IRQ_REMAP_XAPIC_MODE)
  1325. pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n");
  1326. else if (ret < 0)
  1327. pr_info("x2apic not enabled, IRQ remapping init failed\n");
  1328. }
  1329. #ifdef CONFIG_X86_64
  1330. /*
  1331. * Detect and enable local APICs on non-SMP boards.
  1332. * Original code written by Keir Fraser.
  1333. * On AMD64 we trust the BIOS - if it says no APIC it is likely
  1334. * not correctly set up (usually the APIC timer won't work etc.)
  1335. */
  1336. static int __init detect_init_APIC(void)
  1337. {
  1338. if (!cpu_has_apic) {
  1339. pr_info("No local APIC present\n");
  1340. return -1;
  1341. }
  1342. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1343. return 0;
  1344. }
  1345. #else
  1346. static int __init apic_verify(void)
  1347. {
  1348. u32 features, h, l;
  1349. /*
  1350. * The APIC feature bit should now be enabled
  1351. * in `cpuid'
  1352. */
  1353. features = cpuid_edx(1);
  1354. if (!(features & (1 << X86_FEATURE_APIC))) {
  1355. pr_warning("Could not enable APIC!\n");
  1356. return -1;
  1357. }
  1358. set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
  1359. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1360. /* The BIOS may have set up the APIC at some other address */
  1361. rdmsr(MSR_IA32_APICBASE, l, h);
  1362. if (l & MSR_IA32_APICBASE_ENABLE)
  1363. mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
  1364. pr_info("Found and enabled local APIC!\n");
  1365. return 0;
  1366. }
  1367. int __init apic_force_enable(unsigned long addr)
  1368. {
  1369. u32 h, l;
  1370. if (disable_apic)
  1371. return -1;
  1372. /*
  1373. * Some BIOSes disable the local APIC in the APIC_BASE
  1374. * MSR. This can only be done in software for Intel P6 or later
  1375. * and AMD K7 (Model > 1) or later.
  1376. */
  1377. rdmsr(MSR_IA32_APICBASE, l, h);
  1378. if (!(l & MSR_IA32_APICBASE_ENABLE)) {
  1379. pr_info("Local APIC disabled by BIOS -- reenabling.\n");
  1380. l &= ~MSR_IA32_APICBASE_BASE;
  1381. l |= MSR_IA32_APICBASE_ENABLE | addr;
  1382. wrmsr(MSR_IA32_APICBASE, l, h);
  1383. enabled_via_apicbase = 1;
  1384. }
  1385. return apic_verify();
  1386. }
  1387. /*
  1388. * Detect and initialize APIC
  1389. */
  1390. static int __init detect_init_APIC(void)
  1391. {
  1392. /* Disabled by kernel option? */
  1393. if (disable_apic)
  1394. return -1;
  1395. switch (boot_cpu_data.x86_vendor) {
  1396. case X86_VENDOR_AMD:
  1397. if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
  1398. (boot_cpu_data.x86 >= 15))
  1399. break;
  1400. goto no_apic;
  1401. case X86_VENDOR_INTEL:
  1402. if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
  1403. (boot_cpu_data.x86 == 5 && cpu_has_apic))
  1404. break;
  1405. goto no_apic;
  1406. default:
  1407. goto no_apic;
  1408. }
  1409. if (!cpu_has_apic) {
  1410. /*
  1411. * Over-ride BIOS and try to enable the local APIC only if
  1412. * "lapic" specified.
  1413. */
  1414. if (!force_enable_local_apic) {
  1415. pr_info("Local APIC disabled by BIOS -- "
  1416. "you can enable it with \"lapic\"\n");
  1417. return -1;
  1418. }
  1419. if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
  1420. return -1;
  1421. } else {
  1422. if (apic_verify())
  1423. return -1;
  1424. }
  1425. apic_pm_activate();
  1426. return 0;
  1427. no_apic:
  1428. pr_info("No local APIC present or hardware disabled\n");
  1429. return -1;
  1430. }
  1431. #endif
  1432. /**
  1433. * init_apic_mappings - initialize APIC mappings
  1434. */
  1435. void __init init_apic_mappings(void)
  1436. {
  1437. unsigned int new_apicid;
  1438. if (x2apic_mode) {
  1439. boot_cpu_physical_apicid = read_apic_id();
  1440. return;
  1441. }
  1442. /* If no local APIC can be found return early */
  1443. if (!smp_found_config && detect_init_APIC()) {
  1444. /* lets NOP'ify apic operations */
  1445. pr_info("APIC: disable apic facility\n");
  1446. apic_disable();
  1447. } else {
  1448. apic_phys = mp_lapic_addr;
  1449. /*
  1450. * acpi lapic path already maps that address in
  1451. * acpi_register_lapic_address()
  1452. */
  1453. if (!acpi_lapic && !smp_found_config)
  1454. register_lapic_address(apic_phys);
  1455. }
  1456. /*
  1457. * Fetch the APIC ID of the BSP in case we have a
  1458. * default configuration (or the MP table is broken).
  1459. */
  1460. new_apicid = read_apic_id();
  1461. if (boot_cpu_physical_apicid != new_apicid) {
  1462. boot_cpu_physical_apicid = new_apicid;
  1463. /*
  1464. * yeah -- we lie about apic_version
  1465. * in case if apic was disabled via boot option
  1466. * but it's not a problem for SMP compiled kernel
  1467. * since smp_sanity_check is prepared for such a case
  1468. * and disable smp mode
  1469. */
  1470. apic_version[new_apicid] =
  1471. GET_APIC_VERSION(apic_read(APIC_LVR));
  1472. }
  1473. }
  1474. void __init register_lapic_address(unsigned long address)
  1475. {
  1476. mp_lapic_addr = address;
  1477. if (!x2apic_mode) {
  1478. set_fixmap_nocache(FIX_APIC_BASE, address);
  1479. apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
  1480. APIC_BASE, mp_lapic_addr);
  1481. }
  1482. if (boot_cpu_physical_apicid == -1U) {
  1483. boot_cpu_physical_apicid = read_apic_id();
  1484. apic_version[boot_cpu_physical_apicid] =
  1485. GET_APIC_VERSION(apic_read(APIC_LVR));
  1486. }
  1487. }
  1488. /*
  1489. * This initializes the IO-APIC and APIC hardware if this is
  1490. * a UP kernel.
  1491. */
  1492. int apic_version[MAX_LOCAL_APIC];
  1493. int __init APIC_init_uniprocessor(void)
  1494. {
  1495. if (disable_apic) {
  1496. pr_info("Apic disabled\n");
  1497. return -1;
  1498. }
  1499. #ifdef CONFIG_X86_64
  1500. if (!cpu_has_apic) {
  1501. disable_apic = 1;
  1502. pr_info("Apic disabled by BIOS\n");
  1503. return -1;
  1504. }
  1505. #else
  1506. if (!smp_found_config && !cpu_has_apic)
  1507. return -1;
  1508. /*
  1509. * Complain if the BIOS pretends there is one.
  1510. */
  1511. if (!cpu_has_apic &&
  1512. APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
  1513. pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
  1514. boot_cpu_physical_apicid);
  1515. return -1;
  1516. }
  1517. #endif
  1518. default_setup_apic_routing();
  1519. verify_local_APIC();
  1520. connect_bsp_APIC();
  1521. #ifdef CONFIG_X86_64
  1522. apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
  1523. #else
  1524. /*
  1525. * Hack: In case of kdump, after a crash, kernel might be booting
  1526. * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
  1527. * might be zero if read from MP tables. Get it from LAPIC.
  1528. */
  1529. # ifdef CONFIG_CRASH_DUMP
  1530. boot_cpu_physical_apicid = read_apic_id();
  1531. # endif
  1532. #endif
  1533. physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
  1534. setup_local_APIC();
  1535. #ifdef CONFIG_X86_IO_APIC
  1536. /*
  1537. * Now enable IO-APICs, actually call clear_IO_APIC
  1538. * We need clear_IO_APIC before enabling error vector
  1539. */
  1540. if (!skip_ioapic_setup && nr_ioapics)
  1541. enable_IO_APIC();
  1542. #endif
  1543. bsp_end_local_APIC_setup();
  1544. #ifdef CONFIG_X86_IO_APIC
  1545. if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
  1546. setup_IO_APIC();
  1547. else {
  1548. nr_ioapics = 0;
  1549. }
  1550. #endif
  1551. x86_init.timers.setup_percpu_clockev();
  1552. return 0;
  1553. }
  1554. /*
  1555. * Local APIC interrupts
  1556. */
  1557. /*
  1558. * This interrupt should _never_ happen with our APIC/SMP architecture
  1559. */
  1560. void smp_spurious_interrupt(struct pt_regs *regs)
  1561. {
  1562. u32 v;
  1563. exit_idle();
  1564. irq_enter();
  1565. /*
  1566. * Check if this really is a spurious interrupt and ACK it
  1567. * if it is a vectored one. Just in case...
  1568. * Spurious interrupts should not be ACKed.
  1569. */
  1570. v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
  1571. if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
  1572. ack_APIC_irq();
  1573. inc_irq_stat(irq_spurious_count);
  1574. /* see sw-dev-man vol 3, chapter 7.4.13.5 */
  1575. pr_info("spurious APIC interrupt on CPU#%d, "
  1576. "should never happen.\n", smp_processor_id());
  1577. irq_exit();
  1578. }
  1579. /*
  1580. * This interrupt should never happen with our APIC/SMP architecture
  1581. */
  1582. void smp_error_interrupt(struct pt_regs *regs)
  1583. {
  1584. u32 v0, v1;
  1585. u32 i = 0;
  1586. static const char * const error_interrupt_reason[] = {
  1587. "Send CS error", /* APIC Error Bit 0 */
  1588. "Receive CS error", /* APIC Error Bit 1 */
  1589. "Send accept error", /* APIC Error Bit 2 */
  1590. "Receive accept error", /* APIC Error Bit 3 */
  1591. "Redirectable IPI", /* APIC Error Bit 4 */
  1592. "Send illegal vector", /* APIC Error Bit 5 */
  1593. "Received illegal vector", /* APIC Error Bit 6 */
  1594. "Illegal register address", /* APIC Error Bit 7 */
  1595. };
  1596. exit_idle();
  1597. irq_enter();
  1598. /* First tickle the hardware, only then report what went on. -- REW */
  1599. v0 = apic_read(APIC_ESR);
  1600. apic_write(APIC_ESR, 0);
  1601. v1 = apic_read(APIC_ESR);
  1602. ack_APIC_irq();
  1603. atomic_inc(&irq_err_count);
  1604. apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x(%02x)",
  1605. smp_processor_id(), v0 , v1);
  1606. v1 = v1 & 0xff;
  1607. while (v1) {
  1608. if (v1 & 0x1)
  1609. apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
  1610. i++;
  1611. v1 >>= 1;
  1612. };
  1613. apic_printk(APIC_DEBUG, KERN_CONT "\n");
  1614. irq_exit();
  1615. }
  1616. /**
  1617. * connect_bsp_APIC - attach the APIC to the interrupt system
  1618. */
  1619. void __init connect_bsp_APIC(void)
  1620. {
  1621. #ifdef CONFIG_X86_32
  1622. if (pic_mode) {
  1623. /*
  1624. * Do not trust the local APIC being empty at bootup.
  1625. */
  1626. clear_local_APIC();
  1627. /*
  1628. * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
  1629. * local APIC to INT and NMI lines.
  1630. */
  1631. apic_printk(APIC_VERBOSE, "leaving PIC mode, "
  1632. "enabling APIC mode.\n");
  1633. imcr_pic_to_apic();
  1634. }
  1635. #endif
  1636. if (apic->enable_apic_mode)
  1637. apic->enable_apic_mode();
  1638. }
  1639. /**
  1640. * disconnect_bsp_APIC - detach the APIC from the interrupt system
  1641. * @virt_wire_setup: indicates, whether virtual wire mode is selected
  1642. *
  1643. * Virtual wire mode is necessary to deliver legacy interrupts even when the
  1644. * APIC is disabled.
  1645. */
  1646. void disconnect_bsp_APIC(int virt_wire_setup)
  1647. {
  1648. unsigned int value;
  1649. #ifdef CONFIG_X86_32
  1650. if (pic_mode) {
  1651. /*
  1652. * Put the board back into PIC mode (has an effect only on
  1653. * certain older boards). Note that APIC interrupts, including
  1654. * IPIs, won't work beyond this point! The only exception are
  1655. * INIT IPIs.
  1656. */
  1657. apic_printk(APIC_VERBOSE, "disabling APIC mode, "
  1658. "entering PIC mode.\n");
  1659. imcr_apic_to_pic();
  1660. return;
  1661. }
  1662. #endif
  1663. /* Go back to Virtual Wire compatibility mode */
  1664. /* For the spurious interrupt use vector F, and enable it */
  1665. value = apic_read(APIC_SPIV);
  1666. value &= ~APIC_VECTOR_MASK;
  1667. value |= APIC_SPIV_APIC_ENABLED;
  1668. value |= 0xf;
  1669. apic_write(APIC_SPIV, value);
  1670. if (!virt_wire_setup) {
  1671. /*
  1672. * For LVT0 make it edge triggered, active high,
  1673. * external and enabled
  1674. */
  1675. value = apic_read(APIC_LVT0);
  1676. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1677. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1678. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1679. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1680. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
  1681. apic_write(APIC_LVT0, value);
  1682. } else {
  1683. /* Disable LVT0 */
  1684. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  1685. }
  1686. /*
  1687. * For LVT1 make it edge triggered, active high,
  1688. * nmi and enabled
  1689. */
  1690. value = apic_read(APIC_LVT1);
  1691. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1692. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1693. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1694. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1695. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
  1696. apic_write(APIC_LVT1, value);
  1697. }
  1698. void __cpuinit generic_processor_info(int apicid, int version)
  1699. {
  1700. int cpu, max = nr_cpu_ids;
  1701. bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
  1702. phys_cpu_present_map);
  1703. /*
  1704. * If boot cpu has not been detected yet, then only allow upto
  1705. * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
  1706. */
  1707. if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
  1708. apicid != boot_cpu_physical_apicid) {
  1709. int thiscpu = max + disabled_cpus - 1;
  1710. pr_warning(
  1711. "ACPI: NR_CPUS/possible_cpus limit of %i almost"
  1712. " reached. Keeping one slot for boot cpu."
  1713. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1714. disabled_cpus++;
  1715. return;
  1716. }
  1717. if (num_processors >= nr_cpu_ids) {
  1718. int thiscpu = max + disabled_cpus;
  1719. pr_warning(
  1720. "ACPI: NR_CPUS/possible_cpus limit of %i reached."
  1721. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1722. disabled_cpus++;
  1723. return;
  1724. }
  1725. num_processors++;
  1726. if (apicid == boot_cpu_physical_apicid) {
  1727. /*
  1728. * x86_bios_cpu_apicid is required to have processors listed
  1729. * in same order as logical cpu numbers. Hence the first
  1730. * entry is BSP, and so on.
  1731. * boot_cpu_init() already hold bit 0 in cpu_present_mask
  1732. * for BSP.
  1733. */
  1734. cpu = 0;
  1735. } else
  1736. cpu = cpumask_next_zero(-1, cpu_present_mask);
  1737. /*
  1738. * Validate version
  1739. */
  1740. if (version == 0x0) {
  1741. pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
  1742. cpu, apicid);
  1743. version = 0x10;
  1744. }
  1745. apic_version[apicid] = version;
  1746. if (version != apic_version[boot_cpu_physical_apicid]) {
  1747. pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
  1748. apic_version[boot_cpu_physical_apicid], cpu, version);
  1749. }
  1750. physid_set(apicid, phys_cpu_present_map);
  1751. if (apicid > max_physical_apicid)
  1752. max_physical_apicid = apicid;
  1753. #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
  1754. early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
  1755. early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
  1756. #endif
  1757. #ifdef CONFIG_X86_32
  1758. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1759. apic->x86_32_early_logical_apicid(cpu);
  1760. #endif
  1761. set_cpu_possible(cpu, true);
  1762. set_cpu_present(cpu, true);
  1763. }
  1764. int hard_smp_processor_id(void)
  1765. {
  1766. return read_apic_id();
  1767. }
  1768. void default_init_apic_ldr(void)
  1769. {
  1770. unsigned long val;
  1771. apic_write(APIC_DFR, APIC_DFR_VALUE);
  1772. val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
  1773. val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
  1774. apic_write(APIC_LDR, val);
  1775. }
  1776. /*
  1777. * Power management
  1778. */
  1779. #ifdef CONFIG_PM
  1780. static struct {
  1781. /*
  1782. * 'active' is true if the local APIC was enabled by us and
  1783. * not the BIOS; this signifies that we are also responsible
  1784. * for disabling it before entering apm/acpi suspend
  1785. */
  1786. int active;
  1787. /* r/w apic fields */
  1788. unsigned int apic_id;
  1789. unsigned int apic_taskpri;
  1790. unsigned int apic_ldr;
  1791. unsigned int apic_dfr;
  1792. unsigned int apic_spiv;
  1793. unsigned int apic_lvtt;
  1794. unsigned int apic_lvtpc;
  1795. unsigned int apic_lvt0;
  1796. unsigned int apic_lvt1;
  1797. unsigned int apic_lvterr;
  1798. unsigned int apic_tmict;
  1799. unsigned int apic_tdcr;
  1800. unsigned int apic_thmr;
  1801. } apic_pm_state;
  1802. static int lapic_suspend(void)
  1803. {
  1804. unsigned long flags;
  1805. int maxlvt;
  1806. if (!apic_pm_state.active)
  1807. return 0;
  1808. maxlvt = lapic_get_maxlvt();
  1809. apic_pm_state.apic_id = apic_read(APIC_ID);
  1810. apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
  1811. apic_pm_state.apic_ldr = apic_read(APIC_LDR);
  1812. apic_pm_state.apic_dfr = apic_read(APIC_DFR);
  1813. apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
  1814. apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
  1815. if (maxlvt >= 4)
  1816. apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
  1817. apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
  1818. apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
  1819. apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
  1820. apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
  1821. apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
  1822. #ifdef CONFIG_X86_THERMAL_VECTOR
  1823. if (maxlvt >= 5)
  1824. apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
  1825. #endif
  1826. local_irq_save(flags);
  1827. disable_local_APIC();
  1828. if (intr_remapping_enabled)
  1829. disable_intr_remapping();
  1830. local_irq_restore(flags);
  1831. return 0;
  1832. }
  1833. static void lapic_resume(void)
  1834. {
  1835. unsigned int l, h;
  1836. unsigned long flags;
  1837. int maxlvt;
  1838. if (!apic_pm_state.active)
  1839. return;
  1840. local_irq_save(flags);
  1841. if (intr_remapping_enabled) {
  1842. /*
  1843. * IO-APIC and PIC have their own resume routines.
  1844. * We just mask them here to make sure the interrupt
  1845. * subsystem is completely quiet while we enable x2apic
  1846. * and interrupt-remapping.
  1847. */
  1848. mask_ioapic_entries();
  1849. legacy_pic->mask_all();
  1850. }
  1851. if (x2apic_mode)
  1852. enable_x2apic();
  1853. else {
  1854. /*
  1855. * Make sure the APICBASE points to the right address
  1856. *
  1857. * FIXME! This will be wrong if we ever support suspend on
  1858. * SMP! We'll need to do this as part of the CPU restore!
  1859. */
  1860. rdmsr(MSR_IA32_APICBASE, l, h);
  1861. l &= ~MSR_IA32_APICBASE_BASE;
  1862. l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
  1863. wrmsr(MSR_IA32_APICBASE, l, h);
  1864. }
  1865. maxlvt = lapic_get_maxlvt();
  1866. apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
  1867. apic_write(APIC_ID, apic_pm_state.apic_id);
  1868. apic_write(APIC_DFR, apic_pm_state.apic_dfr);
  1869. apic_write(APIC_LDR, apic_pm_state.apic_ldr);
  1870. apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
  1871. apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
  1872. apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
  1873. apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
  1874. #if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
  1875. if (maxlvt >= 5)
  1876. apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
  1877. #endif
  1878. if (maxlvt >= 4)
  1879. apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
  1880. apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
  1881. apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
  1882. apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
  1883. apic_write(APIC_ESR, 0);
  1884. apic_read(APIC_ESR);
  1885. apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
  1886. apic_write(APIC_ESR, 0);
  1887. apic_read(APIC_ESR);
  1888. if (intr_remapping_enabled)
  1889. reenable_intr_remapping(x2apic_mode);
  1890. local_irq_restore(flags);
  1891. }
  1892. /*
  1893. * This device has no shutdown method - fully functioning local APICs
  1894. * are needed on every CPU up until machine_halt/restart/poweroff.
  1895. */
  1896. static struct syscore_ops lapic_syscore_ops = {
  1897. .resume = lapic_resume,
  1898. .suspend = lapic_suspend,
  1899. };
  1900. static void __cpuinit apic_pm_activate(void)
  1901. {
  1902. apic_pm_state.active = 1;
  1903. }
  1904. static int __init init_lapic_sysfs(void)
  1905. {
  1906. /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
  1907. if (cpu_has_apic)
  1908. register_syscore_ops(&lapic_syscore_ops);
  1909. return 0;
  1910. }
  1911. /* local apic needs to resume before other devices access its registers. */
  1912. core_initcall(init_lapic_sysfs);
  1913. #else /* CONFIG_PM */
  1914. static void apic_pm_activate(void) { }
  1915. #endif /* CONFIG_PM */
  1916. #ifdef CONFIG_X86_64
  1917. static int __cpuinit apic_cluster_num(void)
  1918. {
  1919. int i, clusters, zeros;
  1920. unsigned id;
  1921. u16 *bios_cpu_apicid;
  1922. DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);
  1923. bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
  1924. bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
  1925. for (i = 0; i < nr_cpu_ids; i++) {
  1926. /* are we being called early in kernel startup? */
  1927. if (bios_cpu_apicid) {
  1928. id = bios_cpu_apicid[i];
  1929. } else if (i < nr_cpu_ids) {
  1930. if (cpu_present(i))
  1931. id = per_cpu(x86_bios_cpu_apicid, i);
  1932. else
  1933. continue;
  1934. } else
  1935. break;
  1936. if (id != BAD_APICID)
  1937. __set_bit(APIC_CLUSTERID(id), clustermap);
  1938. }
  1939. /* Problem: Partially populated chassis may not have CPUs in some of
  1940. * the APIC clusters they have been allocated. Only present CPUs have
  1941. * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
  1942. * Since clusters are allocated sequentially, count zeros only if
  1943. * they are bounded by ones.
  1944. */
  1945. clusters = 0;
  1946. zeros = 0;
  1947. for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
  1948. if (test_bit(i, clustermap)) {
  1949. clusters += 1 + zeros;
  1950. zeros = 0;
  1951. } else
  1952. ++zeros;
  1953. }
  1954. return clusters;
  1955. }
  1956. static int __cpuinitdata multi_checked;
  1957. static int __cpuinitdata multi;
  1958. static int __cpuinit set_multi(const struct dmi_system_id *d)
  1959. {
  1960. if (multi)
  1961. return 0;
  1962. pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
  1963. multi = 1;
  1964. return 0;
  1965. }
  1966. static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = {
  1967. {
  1968. .callback = set_multi,
  1969. .ident = "IBM System Summit2",
  1970. .matches = {
  1971. DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
  1972. DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
  1973. },
  1974. },
  1975. {}
  1976. };
  1977. static void __cpuinit dmi_check_multi(void)
  1978. {
  1979. if (multi_checked)
  1980. return;
  1981. dmi_check_system(multi_dmi_table);
  1982. multi_checked = 1;
  1983. }
  1984. /*
  1985. * apic_is_clustered_box() -- Check if we can expect good TSC
  1986. *
  1987. * Thus far, the major user of this is IBM's Summit2 series:
  1988. * Clustered boxes may have unsynced TSC problems if they are
  1989. * multi-chassis.
  1990. * Use DMI to check them
  1991. */
  1992. __cpuinit int apic_is_clustered_box(void)
  1993. {
  1994. dmi_check_multi();
  1995. if (multi)
  1996. return 1;
  1997. if (!is_vsmp_box())
  1998. return 0;
  1999. /*
  2000. * ScaleMP vSMPowered boxes have one cluster per board and TSCs are
  2001. * not guaranteed to be synced between boards
  2002. */
  2003. if (apic_cluster_num() > 1)
  2004. return 1;
  2005. return 0;
  2006. }
  2007. #endif
  2008. /*
  2009. * APIC command line parameters
  2010. */
  2011. static int __init setup_disableapic(char *arg)
  2012. {
  2013. disable_apic = 1;
  2014. setup_clear_cpu_cap(X86_FEATURE_APIC);
  2015. return 0;
  2016. }
  2017. early_param("disableapic", setup_disableapic);
  2018. /* same as disableapic, for compatibility */
  2019. static int __init setup_nolapic(char *arg)
  2020. {
  2021. return setup_disableapic(arg);
  2022. }
  2023. early_param("nolapic", setup_nolapic);
  2024. static int __init parse_lapic_timer_c2_ok(char *arg)
  2025. {
  2026. local_apic_timer_c2_ok = 1;
  2027. return 0;
  2028. }
  2029. early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
  2030. static int __init parse_disable_apic_timer(char *arg)
  2031. {
  2032. disable_apic_timer = 1;
  2033. return 0;
  2034. }
  2035. early_param("noapictimer", parse_disable_apic_timer);
  2036. static int __init parse_nolapic_timer(char *arg)
  2037. {
  2038. disable_apic_timer = 1;
  2039. return 0;
  2040. }
  2041. early_param("nolapic_timer", parse_nolapic_timer);
  2042. static int __init apic_set_verbosity(char *arg)
  2043. {
  2044. if (!arg) {
  2045. #ifdef CONFIG_X86_64
  2046. skip_ioapic_setup = 0;
  2047. return 0;
  2048. #endif
  2049. return -EINVAL;
  2050. }
  2051. if (strcmp("debug", arg) == 0)
  2052. apic_verbosity = APIC_DEBUG;
  2053. else if (strcmp("verbose", arg) == 0)
  2054. apic_verbosity = APIC_VERBOSE;
  2055. else {
  2056. pr_warning("APIC Verbosity level %s not recognised"
  2057. " use apic=verbose or apic=debug\n", arg);
  2058. return -EINVAL;
  2059. }
  2060. return 0;
  2061. }
  2062. early_param("apic", apic_set_verbosity);
  2063. static int __init lapic_insert_resource(void)
  2064. {
  2065. if (!apic_phys)
  2066. return -1;
  2067. /* Put local APIC into the resource map. */
  2068. lapic_resource.start = apic_phys;
  2069. lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
  2070. insert_resource(&iomem_resource, &lapic_resource);
  2071. return 0;
  2072. }
  2073. /*
  2074. * need call insert after e820_reserve_resources()
  2075. * that is using request_resource
  2076. */
  2077. late_initcall(lapic_insert_resource);