apic.c 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391
  1. /*
  2. * Local APIC handling, local APIC timers
  3. *
  4. * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
  5. *
  6. * Fixes
  7. * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
  8. * thanks to Eric Gilmore
  9. * and Rolf G. Tews
  10. * for testing these extensively.
  11. * Maciej W. Rozycki : Various updates and fixes.
  12. * Mikael Pettersson : Power Management for UP-APIC.
  13. * Pavel Machek and
  14. * Mikael Pettersson : PM converted to driver model.
  15. */
  16. #include <linux/perf_event.h>
  17. #include <linux/kernel_stat.h>
  18. #include <linux/mc146818rtc.h>
  19. #include <linux/acpi_pmtmr.h>
  20. #include <linux/clockchips.h>
  21. #include <linux/interrupt.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/ftrace.h>
  24. #include <linux/ioport.h>
  25. #include <linux/module.h>
  26. #include <linux/syscore_ops.h>
  27. #include <linux/delay.h>
  28. #include <linux/timex.h>
  29. #include <linux/i8253.h>
  30. #include <linux/dmar.h>
  31. #include <linux/init.h>
  32. #include <linux/cpu.h>
  33. #include <linux/dmi.h>
  34. #include <linux/smp.h>
  35. #include <linux/mm.h>
  36. #include <asm/perf_event.h>
  37. #include <asm/x86_init.h>
  38. #include <asm/pgalloc.h>
  39. #include <linux/atomic.h>
  40. #include <asm/mpspec.h>
  41. #include <asm/i8259.h>
  42. #include <asm/proto.h>
  43. #include <asm/apic.h>
  44. #include <asm/io_apic.h>
  45. #include <asm/desc.h>
  46. #include <asm/hpet.h>
  47. #include <asm/idle.h>
  48. #include <asm/mtrr.h>
  49. #include <asm/time.h>
  50. #include <asm/smp.h>
  51. #include <asm/mce.h>
  52. #include <asm/tsc.h>
  53. #include <asm/hypervisor.h>
  54. unsigned int num_processors;
  55. unsigned disabled_cpus __cpuinitdata;
  56. /* Processor that is doing the boot up */
  57. unsigned int boot_cpu_physical_apicid = -1U;
  58. /*
  59. * The highest APIC ID seen during enumeration.
  60. */
  61. unsigned int max_physical_apicid;
  62. /*
  63. * Bitmask of physically existing CPUs:
  64. */
  65. physid_mask_t phys_cpu_present_map;
  66. /*
  67. * Map cpu index to physical APIC ID
  68. */
  69. DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
  70. DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
  71. EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
  72. EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  73. #ifdef CONFIG_X86_32
  74. /*
  75. * On x86_32, the mapping between cpu and logical apicid may vary
  76. * depending on apic in use. The following early percpu variable is
  77. * used for the mapping. This is where the behaviors of x86_64 and 32
  78. * actually diverge. Let's keep it ugly for now.
  79. */
  80. DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
  81. /*
  82. * Knob to control our willingness to enable the local APIC.
  83. *
  84. * +1=force-enable
  85. */
  86. static int force_enable_local_apic __initdata;
  87. /*
  88. * APIC command line parameters
  89. */
  90. static int __init parse_lapic(char *arg)
  91. {
  92. force_enable_local_apic = 1;
  93. return 0;
  94. }
  95. early_param("lapic", parse_lapic);
  96. /* Local APIC was disabled by the BIOS and enabled by the kernel */
  97. static int enabled_via_apicbase;
  98. /*
  99. * Handle interrupt mode configuration register (IMCR).
  100. * This register controls whether the interrupt signals
  101. * that reach the BSP come from the master PIC or from the
  102. * local APIC. Before entering Symmetric I/O Mode, either
  103. * the BIOS or the operating system must switch out of
  104. * PIC Mode by changing the IMCR.
  105. */
  106. static inline void imcr_pic_to_apic(void)
  107. {
  108. /* select IMCR register */
  109. outb(0x70, 0x22);
  110. /* NMI and 8259 INTR go through APIC */
  111. outb(0x01, 0x23);
  112. }
  113. static inline void imcr_apic_to_pic(void)
  114. {
  115. /* select IMCR register */
  116. outb(0x70, 0x22);
  117. /* NMI and 8259 INTR go directly to BSP */
  118. outb(0x00, 0x23);
  119. }
  120. #endif
  121. #ifdef CONFIG_X86_64
  122. static int apic_calibrate_pmtmr __initdata;
  123. static __init int setup_apicpmtimer(char *s)
  124. {
  125. apic_calibrate_pmtmr = 1;
  126. notsc_setup(NULL);
  127. return 0;
  128. }
  129. __setup("apicpmtimer", setup_apicpmtimer);
  130. #endif
  131. int x2apic_mode;
  132. #ifdef CONFIG_X86_X2APIC
  133. /* x2apic enabled before OS handover */
  134. static int x2apic_preenabled;
  135. static __init int setup_nox2apic(char *str)
  136. {
  137. if (x2apic_enabled()) {
  138. pr_warning("Bios already enabled x2apic, "
  139. "can't enforce nox2apic");
  140. return 0;
  141. }
  142. setup_clear_cpu_cap(X86_FEATURE_X2APIC);
  143. return 0;
  144. }
  145. early_param("nox2apic", setup_nox2apic);
  146. #endif
  147. unsigned long mp_lapic_addr;
  148. int disable_apic;
  149. /* Disable local APIC timer from the kernel commandline or via dmi quirk */
  150. static int disable_apic_timer __initdata;
  151. /* Local APIC timer works in C2 */
  152. int local_apic_timer_c2_ok;
  153. EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
  154. int first_system_vector = 0xfe;
  155. /*
  156. * Debug level, exported for io_apic.c
  157. */
  158. unsigned int apic_verbosity;
  159. int pic_mode;
  160. /* Have we found an MP table */
  161. int smp_found_config;
  162. static struct resource lapic_resource = {
  163. .name = "Local APIC",
  164. .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
  165. };
  166. static unsigned int calibration_result;
  167. static void apic_pm_activate(void);
  168. static unsigned long apic_phys;
  169. /*
  170. * Get the LAPIC version
  171. */
  172. static inline int lapic_get_version(void)
  173. {
  174. return GET_APIC_VERSION(apic_read(APIC_LVR));
  175. }
  176. /*
  177. * Check, if the APIC is integrated or a separate chip
  178. */
  179. static inline int lapic_is_integrated(void)
  180. {
  181. #ifdef CONFIG_X86_64
  182. return 1;
  183. #else
  184. return APIC_INTEGRATED(lapic_get_version());
  185. #endif
  186. }
  187. /*
  188. * Check, whether this is a modern or a first generation APIC
  189. */
  190. static int modern_apic(void)
  191. {
  192. /* AMD systems use old APIC versions, so check the CPU */
  193. if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
  194. boot_cpu_data.x86 >= 0xf)
  195. return 1;
  196. return lapic_get_version() >= 0x14;
  197. }
  198. /*
  199. * right after this call apic become NOOP driven
  200. * so apic->write/read doesn't do anything
  201. */
  202. static void __init apic_disable(void)
  203. {
  204. pr_info("APIC: switched to apic NOOP\n");
  205. apic = &apic_noop;
  206. }
  207. void native_apic_wait_icr_idle(void)
  208. {
  209. while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
  210. cpu_relax();
  211. }
  212. u32 native_safe_apic_wait_icr_idle(void)
  213. {
  214. u32 send_status;
  215. int timeout;
  216. timeout = 0;
  217. do {
  218. send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
  219. if (!send_status)
  220. break;
  221. udelay(100);
  222. } while (timeout++ < 1000);
  223. return send_status;
  224. }
  225. void native_apic_icr_write(u32 low, u32 id)
  226. {
  227. apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
  228. apic_write(APIC_ICR, low);
  229. }
  230. u64 native_apic_icr_read(void)
  231. {
  232. u32 icr1, icr2;
  233. icr2 = apic_read(APIC_ICR2);
  234. icr1 = apic_read(APIC_ICR);
  235. return icr1 | ((u64)icr2 << 32);
  236. }
  237. #ifdef CONFIG_X86_32
  238. /**
  239. * get_physical_broadcast - Get number of physical broadcast IDs
  240. */
  241. int get_physical_broadcast(void)
  242. {
  243. return modern_apic() ? 0xff : 0xf;
  244. }
  245. #endif
  246. /**
  247. * lapic_get_maxlvt - get the maximum number of local vector table entries
  248. */
  249. int lapic_get_maxlvt(void)
  250. {
  251. unsigned int v;
  252. v = apic_read(APIC_LVR);
  253. /*
  254. * - we always have APIC integrated on 64bit mode
  255. * - 82489DXs do not report # of LVT entries
  256. */
  257. return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
  258. }
  259. /*
  260. * Local APIC timer
  261. */
  262. /* Clock divisor */
  263. #define APIC_DIVISOR 16
  264. /*
  265. * This function sets up the local APIC timer, with a timeout of
  266. * 'clocks' APIC bus clock. During calibration we actually call
  267. * this function twice on the boot CPU, once with a bogus timeout
  268. * value, second time for real. The other (noncalibrating) CPUs
  269. * call this function only once, with the real, calibrated value.
  270. *
  271. * We do reads before writes even if unnecessary, to get around the
  272. * P5 APIC double write bug.
  273. */
  274. static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
  275. {
  276. unsigned int lvtt_value, tmp_value;
  277. lvtt_value = LOCAL_TIMER_VECTOR;
  278. if (!oneshot)
  279. lvtt_value |= APIC_LVT_TIMER_PERIODIC;
  280. if (!lapic_is_integrated())
  281. lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
  282. if (!irqen)
  283. lvtt_value |= APIC_LVT_MASKED;
  284. apic_write(APIC_LVTT, lvtt_value);
  285. /*
  286. * Divide PICLK by 16
  287. */
  288. tmp_value = apic_read(APIC_TDCR);
  289. apic_write(APIC_TDCR,
  290. (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
  291. APIC_TDR_DIV_16);
  292. if (!oneshot)
  293. apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
  294. }
  295. /*
  296. * Setup extended LVT, AMD specific
  297. *
  298. * Software should use the LVT offsets the BIOS provides. The offsets
  299. * are determined by the subsystems using it like those for MCE
  300. * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
  301. * are supported. Beginning with family 10h at least 4 offsets are
  302. * available.
  303. *
  304. * Since the offsets must be consistent for all cores, we keep track
  305. * of the LVT offsets in software and reserve the offset for the same
  306. * vector also to be used on other cores. An offset is freed by
  307. * setting the entry to APIC_EILVT_MASKED.
  308. *
  309. * If the BIOS is right, there should be no conflicts. Otherwise a
  310. * "[Firmware Bug]: ..." error message is generated. However, if
  311. * software does not properly determines the offsets, it is not
  312. * necessarily a BIOS bug.
  313. */
  314. static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
  315. static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
  316. {
  317. return (old & APIC_EILVT_MASKED)
  318. || (new == APIC_EILVT_MASKED)
  319. || ((new & ~APIC_EILVT_MASKED) == old);
  320. }
  321. static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
  322. {
  323. unsigned int rsvd; /* 0: uninitialized */
  324. if (offset >= APIC_EILVT_NR_MAX)
  325. return ~0;
  326. rsvd = atomic_read(&eilvt_offsets[offset]) & ~APIC_EILVT_MASKED;
  327. do {
  328. if (rsvd &&
  329. !eilvt_entry_is_changeable(rsvd, new))
  330. /* may not change if vectors are different */
  331. return rsvd;
  332. rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
  333. } while (rsvd != new);
  334. return new;
  335. }
  336. /*
  337. * If mask=1, the LVT entry does not generate interrupts while mask=0
  338. * enables the vector. See also the BKDGs. Must be called with
  339. * preemption disabled.
  340. */
  341. int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
  342. {
  343. unsigned long reg = APIC_EILVTn(offset);
  344. unsigned int new, old, reserved;
  345. new = (mask << 16) | (msg_type << 8) | vector;
  346. old = apic_read(reg);
  347. reserved = reserve_eilvt_offset(offset, new);
  348. if (reserved != new) {
  349. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  350. "vector 0x%x, but the register is already in use for "
  351. "vector 0x%x on another cpu\n",
  352. smp_processor_id(), reg, offset, new, reserved);
  353. return -EINVAL;
  354. }
  355. if (!eilvt_entry_is_changeable(old, new)) {
  356. pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
  357. "vector 0x%x, but the register is already in use for "
  358. "vector 0x%x on this cpu\n",
  359. smp_processor_id(), reg, offset, new, old);
  360. return -EBUSY;
  361. }
  362. apic_write(reg, new);
  363. return 0;
  364. }
  365. EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
  366. /*
  367. * Program the next event, relative to now
  368. */
  369. static int lapic_next_event(unsigned long delta,
  370. struct clock_event_device *evt)
  371. {
  372. apic_write(APIC_TMICT, delta);
  373. return 0;
  374. }
  375. /*
  376. * Setup the lapic timer in periodic or oneshot mode
  377. */
  378. static void lapic_timer_setup(enum clock_event_mode mode,
  379. struct clock_event_device *evt)
  380. {
  381. unsigned long flags;
  382. unsigned int v;
  383. /* Lapic used as dummy for broadcast ? */
  384. if (evt->features & CLOCK_EVT_FEAT_DUMMY)
  385. return;
  386. local_irq_save(flags);
  387. switch (mode) {
  388. case CLOCK_EVT_MODE_PERIODIC:
  389. case CLOCK_EVT_MODE_ONESHOT:
  390. __setup_APIC_LVTT(calibration_result,
  391. mode != CLOCK_EVT_MODE_PERIODIC, 1);
  392. break;
  393. case CLOCK_EVT_MODE_UNUSED:
  394. case CLOCK_EVT_MODE_SHUTDOWN:
  395. v = apic_read(APIC_LVTT);
  396. v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  397. apic_write(APIC_LVTT, v);
  398. apic_write(APIC_TMICT, 0);
  399. break;
  400. case CLOCK_EVT_MODE_RESUME:
  401. /* Nothing to do here */
  402. break;
  403. }
  404. local_irq_restore(flags);
  405. }
  406. /*
  407. * Local APIC timer broadcast function
  408. */
  409. static void lapic_timer_broadcast(const struct cpumask *mask)
  410. {
  411. #ifdef CONFIG_SMP
  412. apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
  413. #endif
  414. }
  415. /*
  416. * The local apic timer can be used for any function which is CPU local.
  417. */
  418. static struct clock_event_device lapic_clockevent = {
  419. .name = "lapic",
  420. .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
  421. | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
  422. .shift = 32,
  423. .set_mode = lapic_timer_setup,
  424. .set_next_event = lapic_next_event,
  425. .broadcast = lapic_timer_broadcast,
  426. .rating = 100,
  427. .irq = -1,
  428. };
  429. static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
  430. /*
  431. * Setup the local APIC timer for this CPU. Copy the initialized values
  432. * of the boot CPU and register the clock event in the framework.
  433. */
  434. static void __cpuinit setup_APIC_timer(void)
  435. {
  436. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  437. if (this_cpu_has(X86_FEATURE_ARAT)) {
  438. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
  439. /* Make LAPIC timer preferrable over percpu HPET */
  440. lapic_clockevent.rating = 150;
  441. }
  442. memcpy(levt, &lapic_clockevent, sizeof(*levt));
  443. levt->cpumask = cpumask_of(smp_processor_id());
  444. clockevents_register_device(levt);
  445. }
  446. /*
  447. * In this functions we calibrate APIC bus clocks to the external timer.
  448. *
  449. * We want to do the calibration only once since we want to have local timer
  450. * irqs syncron. CPUs connected by the same APIC bus have the very same bus
  451. * frequency.
  452. *
  453. * This was previously done by reading the PIT/HPET and waiting for a wrap
  454. * around to find out, that a tick has elapsed. I have a box, where the PIT
  455. * readout is broken, so it never gets out of the wait loop again. This was
  456. * also reported by others.
  457. *
  458. * Monitoring the jiffies value is inaccurate and the clockevents
  459. * infrastructure allows us to do a simple substitution of the interrupt
  460. * handler.
  461. *
  462. * The calibration routine also uses the pm_timer when possible, as the PIT
  463. * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
  464. * back to normal later in the boot process).
  465. */
  466. #define LAPIC_CAL_LOOPS (HZ/10)
  467. static __initdata int lapic_cal_loops = -1;
  468. static __initdata long lapic_cal_t1, lapic_cal_t2;
  469. static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
  470. static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
  471. static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
  472. /*
  473. * Temporary interrupt handler.
  474. */
  475. static void __init lapic_cal_handler(struct clock_event_device *dev)
  476. {
  477. unsigned long long tsc = 0;
  478. long tapic = apic_read(APIC_TMCCT);
  479. unsigned long pm = acpi_pm_read_early();
  480. if (cpu_has_tsc)
  481. rdtscll(tsc);
  482. switch (lapic_cal_loops++) {
  483. case 0:
  484. lapic_cal_t1 = tapic;
  485. lapic_cal_tsc1 = tsc;
  486. lapic_cal_pm1 = pm;
  487. lapic_cal_j1 = jiffies;
  488. break;
  489. case LAPIC_CAL_LOOPS:
  490. lapic_cal_t2 = tapic;
  491. lapic_cal_tsc2 = tsc;
  492. if (pm < lapic_cal_pm1)
  493. pm += ACPI_PM_OVRRUN;
  494. lapic_cal_pm2 = pm;
  495. lapic_cal_j2 = jiffies;
  496. break;
  497. }
  498. }
  499. static int __init
  500. calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
  501. {
  502. const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
  503. const long pm_thresh = pm_100ms / 100;
  504. unsigned long mult;
  505. u64 res;
  506. #ifndef CONFIG_X86_PM_TIMER
  507. return -1;
  508. #endif
  509. apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
  510. /* Check, if the PM timer is available */
  511. if (!deltapm)
  512. return -1;
  513. mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
  514. if (deltapm > (pm_100ms - pm_thresh) &&
  515. deltapm < (pm_100ms + pm_thresh)) {
  516. apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
  517. return 0;
  518. }
  519. res = (((u64)deltapm) * mult) >> 22;
  520. do_div(res, 1000000);
  521. pr_warning("APIC calibration not consistent "
  522. "with PM-Timer: %ldms instead of 100ms\n",(long)res);
  523. /* Correct the lapic counter value */
  524. res = (((u64)(*delta)) * pm_100ms);
  525. do_div(res, deltapm);
  526. pr_info("APIC delta adjusted to PM-Timer: "
  527. "%lu (%ld)\n", (unsigned long)res, *delta);
  528. *delta = (long)res;
  529. /* Correct the tsc counter value */
  530. if (cpu_has_tsc) {
  531. res = (((u64)(*deltatsc)) * pm_100ms);
  532. do_div(res, deltapm);
  533. apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
  534. "PM-Timer: %lu (%ld)\n",
  535. (unsigned long)res, *deltatsc);
  536. *deltatsc = (long)res;
  537. }
  538. return 0;
  539. }
  540. static int __init calibrate_APIC_clock(void)
  541. {
  542. struct clock_event_device *levt = &__get_cpu_var(lapic_events);
  543. void (*real_handler)(struct clock_event_device *dev);
  544. unsigned long deltaj;
  545. long delta, deltatsc;
  546. int pm_referenced = 0;
  547. local_irq_disable();
  548. /* Replace the global interrupt handler */
  549. real_handler = global_clock_event->event_handler;
  550. global_clock_event->event_handler = lapic_cal_handler;
  551. /*
  552. * Setup the APIC counter to maximum. There is no way the lapic
  553. * can underflow in the 100ms detection time frame
  554. */
  555. __setup_APIC_LVTT(0xffffffff, 0, 0);
  556. /* Let the interrupts run */
  557. local_irq_enable();
  558. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  559. cpu_relax();
  560. local_irq_disable();
  561. /* Restore the real event handler */
  562. global_clock_event->event_handler = real_handler;
  563. /* Build delta t1-t2 as apic timer counts down */
  564. delta = lapic_cal_t1 - lapic_cal_t2;
  565. apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
  566. deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
  567. /* we trust the PM based calibration if possible */
  568. pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
  569. &delta, &deltatsc);
  570. /* Calculate the scaled math multiplication factor */
  571. lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
  572. lapic_clockevent.shift);
  573. lapic_clockevent.max_delta_ns =
  574. clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
  575. lapic_clockevent.min_delta_ns =
  576. clockevent_delta2ns(0xF, &lapic_clockevent);
  577. calibration_result = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
  578. apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
  579. apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
  580. apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
  581. calibration_result);
  582. if (cpu_has_tsc) {
  583. apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
  584. "%ld.%04ld MHz.\n",
  585. (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
  586. (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
  587. }
  588. apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
  589. "%u.%04u MHz.\n",
  590. calibration_result / (1000000 / HZ),
  591. calibration_result % (1000000 / HZ));
  592. /*
  593. * Do a sanity check on the APIC calibration result
  594. */
  595. if (calibration_result < (1000000 / HZ)) {
  596. local_irq_enable();
  597. pr_warning("APIC frequency too slow, disabling apic timer\n");
  598. return -1;
  599. }
  600. levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
  601. /*
  602. * PM timer calibration failed or not turned on
  603. * so lets try APIC timer based calibration
  604. */
  605. if (!pm_referenced) {
  606. apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
  607. /*
  608. * Setup the apic timer manually
  609. */
  610. levt->event_handler = lapic_cal_handler;
  611. lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
  612. lapic_cal_loops = -1;
  613. /* Let the interrupts run */
  614. local_irq_enable();
  615. while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
  616. cpu_relax();
  617. /* Stop the lapic timer */
  618. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
  619. /* Jiffies delta */
  620. deltaj = lapic_cal_j2 - lapic_cal_j1;
  621. apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
  622. /* Check, if the jiffies result is consistent */
  623. if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
  624. apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
  625. else
  626. levt->features |= CLOCK_EVT_FEAT_DUMMY;
  627. } else
  628. local_irq_enable();
  629. if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
  630. pr_warning("APIC timer disabled due to verification failure\n");
  631. return -1;
  632. }
  633. return 0;
  634. }
  635. /*
  636. * Setup the boot APIC
  637. *
  638. * Calibrate and verify the result.
  639. */
  640. void __init setup_boot_APIC_clock(void)
  641. {
  642. /*
  643. * The local apic timer can be disabled via the kernel
  644. * commandline or from the CPU detection code. Register the lapic
  645. * timer as a dummy clock event source on SMP systems, so the
  646. * broadcast mechanism is used. On UP systems simply ignore it.
  647. */
  648. if (disable_apic_timer) {
  649. pr_info("Disabling APIC timer\n");
  650. /* No broadcast on UP ! */
  651. if (num_possible_cpus() > 1) {
  652. lapic_clockevent.mult = 1;
  653. setup_APIC_timer();
  654. }
  655. return;
  656. }
  657. apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
  658. "calibrating APIC timer ...\n");
  659. if (calibrate_APIC_clock()) {
  660. /* No broadcast on UP ! */
  661. if (num_possible_cpus() > 1)
  662. setup_APIC_timer();
  663. return;
  664. }
  665. /*
  666. * If nmi_watchdog is set to IO_APIC, we need the
  667. * PIT/HPET going. Otherwise register lapic as a dummy
  668. * device.
  669. */
  670. lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
  671. /* Setup the lapic or request the broadcast */
  672. setup_APIC_timer();
  673. }
  674. void __cpuinit setup_secondary_APIC_clock(void)
  675. {
  676. setup_APIC_timer();
  677. }
  678. /*
  679. * The guts of the apic timer interrupt
  680. */
  681. static void local_apic_timer_interrupt(void)
  682. {
  683. int cpu = smp_processor_id();
  684. struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
  685. /*
  686. * Normally we should not be here till LAPIC has been initialized but
  687. * in some cases like kdump, its possible that there is a pending LAPIC
  688. * timer interrupt from previous kernel's context and is delivered in
  689. * new kernel the moment interrupts are enabled.
  690. *
  691. * Interrupts are enabled early and LAPIC is setup much later, hence
  692. * its possible that when we get here evt->event_handler is NULL.
  693. * Check for event_handler being NULL and discard the interrupt as
  694. * spurious.
  695. */
  696. if (!evt->event_handler) {
  697. pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
  698. /* Switch it off */
  699. lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
  700. return;
  701. }
  702. /*
  703. * the NMI deadlock-detector uses this.
  704. */
  705. inc_irq_stat(apic_timer_irqs);
  706. evt->event_handler(evt);
  707. }
  708. /*
  709. * Local APIC timer interrupt. This is the most natural way for doing
  710. * local interrupts, but local timer interrupts can be emulated by
  711. * broadcast interrupts too. [in case the hw doesn't support APIC timers]
  712. *
  713. * [ if a single-CPU system runs an SMP kernel then we call the local
  714. * interrupt as well. Thus we cannot inline the local irq ... ]
  715. */
  716. void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
  717. {
  718. struct pt_regs *old_regs = set_irq_regs(regs);
  719. /*
  720. * NOTE! We'd better ACK the irq immediately,
  721. * because timer handling can be slow.
  722. */
  723. ack_APIC_irq();
  724. /*
  725. * update_process_times() expects us to have done irq_enter().
  726. * Besides, if we don't timer interrupts ignore the global
  727. * interrupt lock, which is the WrongThing (tm) to do.
  728. */
  729. exit_idle();
  730. irq_enter();
  731. local_apic_timer_interrupt();
  732. irq_exit();
  733. set_irq_regs(old_regs);
  734. }
  735. int setup_profiling_timer(unsigned int multiplier)
  736. {
  737. return -EINVAL;
  738. }
  739. /*
  740. * Local APIC start and shutdown
  741. */
  742. /**
  743. * clear_local_APIC - shutdown the local APIC
  744. *
  745. * This is called, when a CPU is disabled and before rebooting, so the state of
  746. * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
  747. * leftovers during boot.
  748. */
  749. void clear_local_APIC(void)
  750. {
  751. int maxlvt;
  752. u32 v;
  753. /* APIC hasn't been mapped yet */
  754. if (!x2apic_mode && !apic_phys)
  755. return;
  756. maxlvt = lapic_get_maxlvt();
  757. /*
  758. * Masking an LVT entry can trigger a local APIC error
  759. * if the vector is zero. Mask LVTERR first to prevent this.
  760. */
  761. if (maxlvt >= 3) {
  762. v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
  763. apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
  764. }
  765. /*
  766. * Careful: we have to set masks only first to deassert
  767. * any level-triggered sources.
  768. */
  769. v = apic_read(APIC_LVTT);
  770. apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
  771. v = apic_read(APIC_LVT0);
  772. apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
  773. v = apic_read(APIC_LVT1);
  774. apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
  775. if (maxlvt >= 4) {
  776. v = apic_read(APIC_LVTPC);
  777. apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
  778. }
  779. /* lets not touch this if we didn't frob it */
  780. #ifdef CONFIG_X86_THERMAL_VECTOR
  781. if (maxlvt >= 5) {
  782. v = apic_read(APIC_LVTTHMR);
  783. apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
  784. }
  785. #endif
  786. #ifdef CONFIG_X86_MCE_INTEL
  787. if (maxlvt >= 6) {
  788. v = apic_read(APIC_LVTCMCI);
  789. if (!(v & APIC_LVT_MASKED))
  790. apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
  791. }
  792. #endif
  793. /*
  794. * Clean APIC state for other OSs:
  795. */
  796. apic_write(APIC_LVTT, APIC_LVT_MASKED);
  797. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  798. apic_write(APIC_LVT1, APIC_LVT_MASKED);
  799. if (maxlvt >= 3)
  800. apic_write(APIC_LVTERR, APIC_LVT_MASKED);
  801. if (maxlvt >= 4)
  802. apic_write(APIC_LVTPC, APIC_LVT_MASKED);
  803. /* Integrated APIC (!82489DX) ? */
  804. if (lapic_is_integrated()) {
  805. if (maxlvt > 3)
  806. /* Clear ESR due to Pentium errata 3AP and 11AP */
  807. apic_write(APIC_ESR, 0);
  808. apic_read(APIC_ESR);
  809. }
  810. }
  811. /**
  812. * disable_local_APIC - clear and disable the local APIC
  813. */
  814. void disable_local_APIC(void)
  815. {
  816. unsigned int value;
  817. /* APIC hasn't been mapped yet */
  818. if (!x2apic_mode && !apic_phys)
  819. return;
  820. clear_local_APIC();
  821. /*
  822. * Disable APIC (implies clearing of registers
  823. * for 82489DX!).
  824. */
  825. value = apic_read(APIC_SPIV);
  826. value &= ~APIC_SPIV_APIC_ENABLED;
  827. apic_write(APIC_SPIV, value);
  828. #ifdef CONFIG_X86_32
  829. /*
  830. * When LAPIC was disabled by the BIOS and enabled by the kernel,
  831. * restore the disabled state.
  832. */
  833. if (enabled_via_apicbase) {
  834. unsigned int l, h;
  835. rdmsr(MSR_IA32_APICBASE, l, h);
  836. l &= ~MSR_IA32_APICBASE_ENABLE;
  837. wrmsr(MSR_IA32_APICBASE, l, h);
  838. }
  839. #endif
  840. }
  841. /*
  842. * If Linux enabled the LAPIC against the BIOS default disable it down before
  843. * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
  844. * not power-off. Additionally clear all LVT entries before disable_local_APIC
  845. * for the case where Linux didn't enable the LAPIC.
  846. */
  847. void lapic_shutdown(void)
  848. {
  849. unsigned long flags;
  850. if (!cpu_has_apic && !apic_from_smp_config())
  851. return;
  852. local_irq_save(flags);
  853. #ifdef CONFIG_X86_32
  854. if (!enabled_via_apicbase)
  855. clear_local_APIC();
  856. else
  857. #endif
  858. disable_local_APIC();
  859. local_irq_restore(flags);
  860. }
  861. /*
  862. * This is to verify that we're looking at a real local APIC.
  863. * Check these against your board if the CPUs aren't getting
  864. * started for no apparent reason.
  865. */
  866. int __init verify_local_APIC(void)
  867. {
  868. unsigned int reg0, reg1;
  869. /*
  870. * The version register is read-only in a real APIC.
  871. */
  872. reg0 = apic_read(APIC_LVR);
  873. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
  874. apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
  875. reg1 = apic_read(APIC_LVR);
  876. apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
  877. /*
  878. * The two version reads above should print the same
  879. * numbers. If the second one is different, then we
  880. * poke at a non-APIC.
  881. */
  882. if (reg1 != reg0)
  883. return 0;
  884. /*
  885. * Check if the version looks reasonably.
  886. */
  887. reg1 = GET_APIC_VERSION(reg0);
  888. if (reg1 == 0x00 || reg1 == 0xff)
  889. return 0;
  890. reg1 = lapic_get_maxlvt();
  891. if (reg1 < 0x02 || reg1 == 0xff)
  892. return 0;
  893. /*
  894. * The ID register is read/write in a real APIC.
  895. */
  896. reg0 = apic_read(APIC_ID);
  897. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
  898. apic_write(APIC_ID, reg0 ^ apic->apic_id_mask);
  899. reg1 = apic_read(APIC_ID);
  900. apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
  901. apic_write(APIC_ID, reg0);
  902. if (reg1 != (reg0 ^ apic->apic_id_mask))
  903. return 0;
  904. /*
  905. * The next two are just to see if we have sane values.
  906. * They're only really relevant if we're in Virtual Wire
  907. * compatibility mode, but most boxes are anymore.
  908. */
  909. reg0 = apic_read(APIC_LVT0);
  910. apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
  911. reg1 = apic_read(APIC_LVT1);
  912. apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
  913. return 1;
  914. }
  915. /**
  916. * sync_Arb_IDs - synchronize APIC bus arbitration IDs
  917. */
  918. void __init sync_Arb_IDs(void)
  919. {
  920. /*
  921. * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
  922. * needed on AMD.
  923. */
  924. if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
  925. return;
  926. /*
  927. * Wait for idle.
  928. */
  929. apic_wait_icr_idle();
  930. apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
  931. apic_write(APIC_ICR, APIC_DEST_ALLINC |
  932. APIC_INT_LEVELTRIG | APIC_DM_INIT);
  933. }
  934. /*
  935. * An initial setup of the virtual wire mode.
  936. */
  937. void __init init_bsp_APIC(void)
  938. {
  939. unsigned int value;
  940. /*
  941. * Don't do the setup now if we have a SMP BIOS as the
  942. * through-I/O-APIC virtual wire mode might be active.
  943. */
  944. if (smp_found_config || !cpu_has_apic)
  945. return;
  946. /*
  947. * Do not trust the local APIC being empty at bootup.
  948. */
  949. clear_local_APIC();
  950. /*
  951. * Enable APIC.
  952. */
  953. value = apic_read(APIC_SPIV);
  954. value &= ~APIC_VECTOR_MASK;
  955. value |= APIC_SPIV_APIC_ENABLED;
  956. #ifdef CONFIG_X86_32
  957. /* This bit is reserved on P4/Xeon and should be cleared */
  958. if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
  959. (boot_cpu_data.x86 == 15))
  960. value &= ~APIC_SPIV_FOCUS_DISABLED;
  961. else
  962. #endif
  963. value |= APIC_SPIV_FOCUS_DISABLED;
  964. value |= SPURIOUS_APIC_VECTOR;
  965. apic_write(APIC_SPIV, value);
  966. /*
  967. * Set up the virtual wire mode.
  968. */
  969. apic_write(APIC_LVT0, APIC_DM_EXTINT);
  970. value = APIC_DM_NMI;
  971. if (!lapic_is_integrated()) /* 82489DX */
  972. value |= APIC_LVT_LEVEL_TRIGGER;
  973. apic_write(APIC_LVT1, value);
  974. }
  975. static void __cpuinit lapic_setup_esr(void)
  976. {
  977. unsigned int oldvalue, value, maxlvt;
  978. if (!lapic_is_integrated()) {
  979. pr_info("No ESR for 82489DX.\n");
  980. return;
  981. }
  982. if (apic->disable_esr) {
  983. /*
  984. * Something untraceable is creating bad interrupts on
  985. * secondary quads ... for the moment, just leave the
  986. * ESR disabled - we can't do anything useful with the
  987. * errors anyway - mbligh
  988. */
  989. pr_info("Leaving ESR disabled.\n");
  990. return;
  991. }
  992. maxlvt = lapic_get_maxlvt();
  993. if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
  994. apic_write(APIC_ESR, 0);
  995. oldvalue = apic_read(APIC_ESR);
  996. /* enables sending errors */
  997. value = ERROR_APIC_VECTOR;
  998. apic_write(APIC_LVTERR, value);
  999. /*
  1000. * spec says clear errors after enabling vector.
  1001. */
  1002. if (maxlvt > 3)
  1003. apic_write(APIC_ESR, 0);
  1004. value = apic_read(APIC_ESR);
  1005. if (value != oldvalue)
  1006. apic_printk(APIC_VERBOSE, "ESR value before enabling "
  1007. "vector: 0x%08x after: 0x%08x\n",
  1008. oldvalue, value);
  1009. }
  1010. /**
  1011. * setup_local_APIC - setup the local APIC
  1012. *
  1013. * Used to setup local APIC while initializing BSP or bringin up APs.
  1014. * Always called with preemption disabled.
  1015. */
  1016. void __cpuinit setup_local_APIC(void)
  1017. {
  1018. int cpu = smp_processor_id();
  1019. unsigned int value, queued;
  1020. int i, j, acked = 0;
  1021. unsigned long long tsc = 0, ntsc;
  1022. long long max_loops = cpu_khz;
  1023. if (cpu_has_tsc)
  1024. rdtscll(tsc);
  1025. if (disable_apic) {
  1026. disable_ioapic_support();
  1027. return;
  1028. }
  1029. #ifdef CONFIG_X86_32
  1030. /* Pound the ESR really hard over the head with a big hammer - mbligh */
  1031. if (lapic_is_integrated() && apic->disable_esr) {
  1032. apic_write(APIC_ESR, 0);
  1033. apic_write(APIC_ESR, 0);
  1034. apic_write(APIC_ESR, 0);
  1035. apic_write(APIC_ESR, 0);
  1036. }
  1037. #endif
  1038. perf_events_lapic_init();
  1039. /*
  1040. * Double-check whether this APIC is really registered.
  1041. * This is meaningless in clustered apic mode, so we skip it.
  1042. */
  1043. BUG_ON(!apic->apic_id_registered());
  1044. /*
  1045. * Intel recommends to set DFR, LDR and TPR before enabling
  1046. * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
  1047. * document number 292116). So here it goes...
  1048. */
  1049. apic->init_apic_ldr();
  1050. #ifdef CONFIG_X86_32
  1051. /*
  1052. * APIC LDR is initialized. If logical_apicid mapping was
  1053. * initialized during get_smp_config(), make sure it matches the
  1054. * actual value.
  1055. */
  1056. i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
  1057. WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
  1058. /* always use the value from LDR */
  1059. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1060. logical_smp_processor_id();
  1061. /*
  1062. * Some NUMA implementations (NUMAQ) don't initialize apicid to
  1063. * node mapping during NUMA init. Now that logical apicid is
  1064. * guaranteed to be known, give it another chance. This is already
  1065. * a bit too late - percpu allocation has already happened without
  1066. * proper NUMA affinity.
  1067. */
  1068. if (apic->x86_32_numa_cpu_node)
  1069. set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu),
  1070. apic->x86_32_numa_cpu_node(cpu));
  1071. #endif
  1072. /*
  1073. * Set Task Priority to 'accept all'. We never change this
  1074. * later on.
  1075. */
  1076. value = apic_read(APIC_TASKPRI);
  1077. value &= ~APIC_TPRI_MASK;
  1078. apic_write(APIC_TASKPRI, value);
  1079. /*
  1080. * After a crash, we no longer service the interrupts and a pending
  1081. * interrupt from previous kernel might still have ISR bit set.
  1082. *
  1083. * Most probably by now CPU has serviced that pending interrupt and
  1084. * it might not have done the ack_APIC_irq() because it thought,
  1085. * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
  1086. * does not clear the ISR bit and cpu thinks it has already serivced
  1087. * the interrupt. Hence a vector might get locked. It was noticed
  1088. * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
  1089. */
  1090. do {
  1091. queued = 0;
  1092. for (i = APIC_ISR_NR - 1; i >= 0; i--)
  1093. queued |= apic_read(APIC_IRR + i*0x10);
  1094. for (i = APIC_ISR_NR - 1; i >= 0; i--) {
  1095. value = apic_read(APIC_ISR + i*0x10);
  1096. for (j = 31; j >= 0; j--) {
  1097. if (value & (1<<j)) {
  1098. ack_APIC_irq();
  1099. acked++;
  1100. }
  1101. }
  1102. }
  1103. if (acked > 256) {
  1104. printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
  1105. acked);
  1106. break;
  1107. }
  1108. if (cpu_has_tsc) {
  1109. rdtscll(ntsc);
  1110. max_loops = (cpu_khz << 10) - (ntsc - tsc);
  1111. } else
  1112. max_loops--;
  1113. } while (queued && max_loops > 0);
  1114. WARN_ON(max_loops <= 0);
  1115. /*
  1116. * Now that we are all set up, enable the APIC
  1117. */
  1118. value = apic_read(APIC_SPIV);
  1119. value &= ~APIC_VECTOR_MASK;
  1120. /*
  1121. * Enable APIC
  1122. */
  1123. value |= APIC_SPIV_APIC_ENABLED;
  1124. #ifdef CONFIG_X86_32
  1125. /*
  1126. * Some unknown Intel IO/APIC (or APIC) errata is biting us with
  1127. * certain networking cards. If high frequency interrupts are
  1128. * happening on a particular IOAPIC pin, plus the IOAPIC routing
  1129. * entry is masked/unmasked at a high rate as well then sooner or
  1130. * later IOAPIC line gets 'stuck', no more interrupts are received
  1131. * from the device. If focus CPU is disabled then the hang goes
  1132. * away, oh well :-(
  1133. *
  1134. * [ This bug can be reproduced easily with a level-triggered
  1135. * PCI Ne2000 networking cards and PII/PIII processors, dual
  1136. * BX chipset. ]
  1137. */
  1138. /*
  1139. * Actually disabling the focus CPU check just makes the hang less
  1140. * frequent as it makes the interrupt distributon model be more
  1141. * like LRU than MRU (the short-term load is more even across CPUs).
  1142. * See also the comment in end_level_ioapic_irq(). --macro
  1143. */
  1144. /*
  1145. * - enable focus processor (bit==0)
  1146. * - 64bit mode always use processor focus
  1147. * so no need to set it
  1148. */
  1149. value &= ~APIC_SPIV_FOCUS_DISABLED;
  1150. #endif
  1151. /*
  1152. * Set spurious IRQ vector
  1153. */
  1154. value |= SPURIOUS_APIC_VECTOR;
  1155. apic_write(APIC_SPIV, value);
  1156. /*
  1157. * Set up LVT0, LVT1:
  1158. *
  1159. * set up through-local-APIC on the BP's LINT0. This is not
  1160. * strictly necessary in pure symmetric-IO mode, but sometimes
  1161. * we delegate interrupts to the 8259A.
  1162. */
  1163. /*
  1164. * TODO: set up through-local-APIC from through-I/O-APIC? --macro
  1165. */
  1166. value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
  1167. if (!cpu && (pic_mode || !value)) {
  1168. value = APIC_DM_EXTINT;
  1169. apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
  1170. } else {
  1171. value = APIC_DM_EXTINT | APIC_LVT_MASKED;
  1172. apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
  1173. }
  1174. apic_write(APIC_LVT0, value);
  1175. /*
  1176. * only the BP should see the LINT1 NMI signal, obviously.
  1177. */
  1178. if (!cpu)
  1179. value = APIC_DM_NMI;
  1180. else
  1181. value = APIC_DM_NMI | APIC_LVT_MASKED;
  1182. if (!lapic_is_integrated()) /* 82489DX */
  1183. value |= APIC_LVT_LEVEL_TRIGGER;
  1184. apic_write(APIC_LVT1, value);
  1185. #ifdef CONFIG_X86_MCE_INTEL
  1186. /* Recheck CMCI information after local APIC is up on CPU #0 */
  1187. if (!cpu)
  1188. cmci_recheck();
  1189. #endif
  1190. }
  1191. void __cpuinit end_local_APIC_setup(void)
  1192. {
  1193. lapic_setup_esr();
  1194. #ifdef CONFIG_X86_32
  1195. {
  1196. unsigned int value;
  1197. /* Disable the local apic timer */
  1198. value = apic_read(APIC_LVTT);
  1199. value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
  1200. apic_write(APIC_LVTT, value);
  1201. }
  1202. #endif
  1203. apic_pm_activate();
  1204. }
  1205. void __init bsp_end_local_APIC_setup(void)
  1206. {
  1207. end_local_APIC_setup();
  1208. /*
  1209. * Now that local APIC setup is completed for BP, configure the fault
  1210. * handling for interrupt remapping.
  1211. */
  1212. if (intr_remapping_enabled)
  1213. enable_drhd_fault_handling();
  1214. }
  1215. #ifdef CONFIG_X86_X2APIC
  1216. void check_x2apic(void)
  1217. {
  1218. if (x2apic_enabled()) {
  1219. pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
  1220. x2apic_preenabled = x2apic_mode = 1;
  1221. }
  1222. }
  1223. void enable_x2apic(void)
  1224. {
  1225. int msr, msr2;
  1226. if (!x2apic_mode)
  1227. return;
  1228. rdmsr(MSR_IA32_APICBASE, msr, msr2);
  1229. if (!(msr & X2APIC_ENABLE)) {
  1230. printk_once(KERN_INFO "Enabling x2apic\n");
  1231. wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2);
  1232. }
  1233. }
  1234. #endif /* CONFIG_X86_X2APIC */
  1235. int __init enable_IR(void)
  1236. {
  1237. #ifdef CONFIG_INTR_REMAP
  1238. if (!intr_remapping_supported()) {
  1239. pr_debug("intr-remapping not supported\n");
  1240. return 0;
  1241. }
  1242. if (!x2apic_preenabled && skip_ioapic_setup) {
  1243. pr_info("Skipped enabling intr-remap because of skipping "
  1244. "io-apic setup\n");
  1245. return 0;
  1246. }
  1247. if (enable_intr_remapping(x2apic_supported()))
  1248. return 0;
  1249. pr_info("Enabled Interrupt-remapping\n");
  1250. return 1;
  1251. #endif
  1252. return 0;
  1253. }
  1254. void __init enable_IR_x2apic(void)
  1255. {
  1256. unsigned long flags;
  1257. int ret, x2apic_enabled = 0;
  1258. int dmar_table_init_ret;
  1259. dmar_table_init_ret = dmar_table_init();
  1260. if (dmar_table_init_ret && !x2apic_supported())
  1261. return;
  1262. ret = save_ioapic_entries();
  1263. if (ret) {
  1264. pr_info("Saving IO-APIC state failed: %d\n", ret);
  1265. goto out;
  1266. }
  1267. local_irq_save(flags);
  1268. legacy_pic->mask_all();
  1269. mask_ioapic_entries();
  1270. if (dmar_table_init_ret)
  1271. ret = 0;
  1272. else
  1273. ret = enable_IR();
  1274. if (!ret) {
  1275. /* IR is required if there is APIC ID > 255 even when running
  1276. * under KVM
  1277. */
  1278. if (max_physical_apicid > 255 ||
  1279. !hypervisor_x2apic_available())
  1280. goto nox2apic;
  1281. /*
  1282. * without IR all CPUs can be addressed by IOAPIC/MSI
  1283. * only in physical mode
  1284. */
  1285. x2apic_force_phys();
  1286. }
  1287. x2apic_enabled = 1;
  1288. if (x2apic_supported() && !x2apic_mode) {
  1289. x2apic_mode = 1;
  1290. enable_x2apic();
  1291. pr_info("Enabled x2apic\n");
  1292. }
  1293. nox2apic:
  1294. if (!ret) /* IR enabling failed */
  1295. restore_ioapic_entries();
  1296. legacy_pic->restore_mask();
  1297. local_irq_restore(flags);
  1298. out:
  1299. if (x2apic_enabled)
  1300. return;
  1301. if (x2apic_preenabled)
  1302. panic("x2apic: enabled by BIOS but kernel init failed.");
  1303. else if (cpu_has_x2apic)
  1304. pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
  1305. }
  1306. #ifdef CONFIG_X86_64
  1307. /*
  1308. * Detect and enable local APICs on non-SMP boards.
  1309. * Original code written by Keir Fraser.
  1310. * On AMD64 we trust the BIOS - if it says no APIC it is likely
  1311. * not correctly set up (usually the APIC timer won't work etc.)
  1312. */
  1313. static int __init detect_init_APIC(void)
  1314. {
  1315. if (!cpu_has_apic) {
  1316. pr_info("No local APIC present\n");
  1317. return -1;
  1318. }
  1319. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1320. return 0;
  1321. }
  1322. #else
  1323. static int __init apic_verify(void)
  1324. {
  1325. u32 features, h, l;
  1326. /*
  1327. * The APIC feature bit should now be enabled
  1328. * in `cpuid'
  1329. */
  1330. features = cpuid_edx(1);
  1331. if (!(features & (1 << X86_FEATURE_APIC))) {
  1332. pr_warning("Could not enable APIC!\n");
  1333. return -1;
  1334. }
  1335. set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
  1336. mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
  1337. /* The BIOS may have set up the APIC at some other address */
  1338. rdmsr(MSR_IA32_APICBASE, l, h);
  1339. if (l & MSR_IA32_APICBASE_ENABLE)
  1340. mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
  1341. pr_info("Found and enabled local APIC!\n");
  1342. return 0;
  1343. }
  1344. int __init apic_force_enable(unsigned long addr)
  1345. {
  1346. u32 h, l;
  1347. if (disable_apic)
  1348. return -1;
  1349. /*
  1350. * Some BIOSes disable the local APIC in the APIC_BASE
  1351. * MSR. This can only be done in software for Intel P6 or later
  1352. * and AMD K7 (Model > 1) or later.
  1353. */
  1354. rdmsr(MSR_IA32_APICBASE, l, h);
  1355. if (!(l & MSR_IA32_APICBASE_ENABLE)) {
  1356. pr_info("Local APIC disabled by BIOS -- reenabling.\n");
  1357. l &= ~MSR_IA32_APICBASE_BASE;
  1358. l |= MSR_IA32_APICBASE_ENABLE | addr;
  1359. wrmsr(MSR_IA32_APICBASE, l, h);
  1360. enabled_via_apicbase = 1;
  1361. }
  1362. return apic_verify();
  1363. }
  1364. /*
  1365. * Detect and initialize APIC
  1366. */
  1367. static int __init detect_init_APIC(void)
  1368. {
  1369. /* Disabled by kernel option? */
  1370. if (disable_apic)
  1371. return -1;
  1372. switch (boot_cpu_data.x86_vendor) {
  1373. case X86_VENDOR_AMD:
  1374. if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
  1375. (boot_cpu_data.x86 >= 15))
  1376. break;
  1377. goto no_apic;
  1378. case X86_VENDOR_INTEL:
  1379. if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
  1380. (boot_cpu_data.x86 == 5 && cpu_has_apic))
  1381. break;
  1382. goto no_apic;
  1383. default:
  1384. goto no_apic;
  1385. }
  1386. if (!cpu_has_apic) {
  1387. /*
  1388. * Over-ride BIOS and try to enable the local APIC only if
  1389. * "lapic" specified.
  1390. */
  1391. if (!force_enable_local_apic) {
  1392. pr_info("Local APIC disabled by BIOS -- "
  1393. "you can enable it with \"lapic\"\n");
  1394. return -1;
  1395. }
  1396. if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
  1397. return -1;
  1398. } else {
  1399. if (apic_verify())
  1400. return -1;
  1401. }
  1402. apic_pm_activate();
  1403. return 0;
  1404. no_apic:
  1405. pr_info("No local APIC present or hardware disabled\n");
  1406. return -1;
  1407. }
  1408. #endif
  1409. /**
  1410. * init_apic_mappings - initialize APIC mappings
  1411. */
  1412. void __init init_apic_mappings(void)
  1413. {
  1414. unsigned int new_apicid;
  1415. if (x2apic_mode) {
  1416. boot_cpu_physical_apicid = read_apic_id();
  1417. return;
  1418. }
  1419. /* If no local APIC can be found return early */
  1420. if (!smp_found_config && detect_init_APIC()) {
  1421. /* lets NOP'ify apic operations */
  1422. pr_info("APIC: disable apic facility\n");
  1423. apic_disable();
  1424. } else {
  1425. apic_phys = mp_lapic_addr;
  1426. /*
  1427. * acpi lapic path already maps that address in
  1428. * acpi_register_lapic_address()
  1429. */
  1430. if (!acpi_lapic && !smp_found_config)
  1431. register_lapic_address(apic_phys);
  1432. }
  1433. /*
  1434. * Fetch the APIC ID of the BSP in case we have a
  1435. * default configuration (or the MP table is broken).
  1436. */
  1437. new_apicid = read_apic_id();
  1438. if (boot_cpu_physical_apicid != new_apicid) {
  1439. boot_cpu_physical_apicid = new_apicid;
  1440. /*
  1441. * yeah -- we lie about apic_version
  1442. * in case if apic was disabled via boot option
  1443. * but it's not a problem for SMP compiled kernel
  1444. * since smp_sanity_check is prepared for such a case
  1445. * and disable smp mode
  1446. */
  1447. apic_version[new_apicid] =
  1448. GET_APIC_VERSION(apic_read(APIC_LVR));
  1449. }
  1450. }
  1451. void __init register_lapic_address(unsigned long address)
  1452. {
  1453. mp_lapic_addr = address;
  1454. if (!x2apic_mode) {
  1455. set_fixmap_nocache(FIX_APIC_BASE, address);
  1456. apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
  1457. APIC_BASE, mp_lapic_addr);
  1458. }
  1459. if (boot_cpu_physical_apicid == -1U) {
  1460. boot_cpu_physical_apicid = read_apic_id();
  1461. apic_version[boot_cpu_physical_apicid] =
  1462. GET_APIC_VERSION(apic_read(APIC_LVR));
  1463. }
  1464. }
  1465. /*
  1466. * This initializes the IO-APIC and APIC hardware if this is
  1467. * a UP kernel.
  1468. */
  1469. int apic_version[MAX_LOCAL_APIC];
  1470. int __init APIC_init_uniprocessor(void)
  1471. {
  1472. if (disable_apic) {
  1473. pr_info("Apic disabled\n");
  1474. return -1;
  1475. }
  1476. #ifdef CONFIG_X86_64
  1477. if (!cpu_has_apic) {
  1478. disable_apic = 1;
  1479. pr_info("Apic disabled by BIOS\n");
  1480. return -1;
  1481. }
  1482. #else
  1483. if (!smp_found_config && !cpu_has_apic)
  1484. return -1;
  1485. /*
  1486. * Complain if the BIOS pretends there is one.
  1487. */
  1488. if (!cpu_has_apic &&
  1489. APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
  1490. pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
  1491. boot_cpu_physical_apicid);
  1492. return -1;
  1493. }
  1494. #endif
  1495. default_setup_apic_routing();
  1496. verify_local_APIC();
  1497. connect_bsp_APIC();
  1498. #ifdef CONFIG_X86_64
  1499. apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
  1500. #else
  1501. /*
  1502. * Hack: In case of kdump, after a crash, kernel might be booting
  1503. * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
  1504. * might be zero if read from MP tables. Get it from LAPIC.
  1505. */
  1506. # ifdef CONFIG_CRASH_DUMP
  1507. boot_cpu_physical_apicid = read_apic_id();
  1508. # endif
  1509. #endif
  1510. physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
  1511. setup_local_APIC();
  1512. #ifdef CONFIG_X86_IO_APIC
  1513. /*
  1514. * Now enable IO-APICs, actually call clear_IO_APIC
  1515. * We need clear_IO_APIC before enabling error vector
  1516. */
  1517. if (!skip_ioapic_setup && nr_ioapics)
  1518. enable_IO_APIC();
  1519. #endif
  1520. bsp_end_local_APIC_setup();
  1521. #ifdef CONFIG_X86_IO_APIC
  1522. if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
  1523. setup_IO_APIC();
  1524. else {
  1525. nr_ioapics = 0;
  1526. }
  1527. #endif
  1528. x86_init.timers.setup_percpu_clockev();
  1529. return 0;
  1530. }
  1531. /*
  1532. * Local APIC interrupts
  1533. */
  1534. /*
  1535. * This interrupt should _never_ happen with our APIC/SMP architecture
  1536. */
  1537. void smp_spurious_interrupt(struct pt_regs *regs)
  1538. {
  1539. u32 v;
  1540. exit_idle();
  1541. irq_enter();
  1542. /*
  1543. * Check if this really is a spurious interrupt and ACK it
  1544. * if it is a vectored one. Just in case...
  1545. * Spurious interrupts should not be ACKed.
  1546. */
  1547. v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
  1548. if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
  1549. ack_APIC_irq();
  1550. inc_irq_stat(irq_spurious_count);
  1551. /* see sw-dev-man vol 3, chapter 7.4.13.5 */
  1552. pr_info("spurious APIC interrupt on CPU#%d, "
  1553. "should never happen.\n", smp_processor_id());
  1554. irq_exit();
  1555. }
  1556. /*
  1557. * This interrupt should never happen with our APIC/SMP architecture
  1558. */
  1559. void smp_error_interrupt(struct pt_regs *regs)
  1560. {
  1561. u32 v0, v1;
  1562. u32 i = 0;
  1563. static const char * const error_interrupt_reason[] = {
  1564. "Send CS error", /* APIC Error Bit 0 */
  1565. "Receive CS error", /* APIC Error Bit 1 */
  1566. "Send accept error", /* APIC Error Bit 2 */
  1567. "Receive accept error", /* APIC Error Bit 3 */
  1568. "Redirectable IPI", /* APIC Error Bit 4 */
  1569. "Send illegal vector", /* APIC Error Bit 5 */
  1570. "Received illegal vector", /* APIC Error Bit 6 */
  1571. "Illegal register address", /* APIC Error Bit 7 */
  1572. };
  1573. exit_idle();
  1574. irq_enter();
  1575. /* First tickle the hardware, only then report what went on. -- REW */
  1576. v0 = apic_read(APIC_ESR);
  1577. apic_write(APIC_ESR, 0);
  1578. v1 = apic_read(APIC_ESR);
  1579. ack_APIC_irq();
  1580. atomic_inc(&irq_err_count);
  1581. apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x(%02x)",
  1582. smp_processor_id(), v0 , v1);
  1583. v1 = v1 & 0xff;
  1584. while (v1) {
  1585. if (v1 & 0x1)
  1586. apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
  1587. i++;
  1588. v1 >>= 1;
  1589. };
  1590. apic_printk(APIC_DEBUG, KERN_CONT "\n");
  1591. irq_exit();
  1592. }
  1593. /**
  1594. * connect_bsp_APIC - attach the APIC to the interrupt system
  1595. */
  1596. void __init connect_bsp_APIC(void)
  1597. {
  1598. #ifdef CONFIG_X86_32
  1599. if (pic_mode) {
  1600. /*
  1601. * Do not trust the local APIC being empty at bootup.
  1602. */
  1603. clear_local_APIC();
  1604. /*
  1605. * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
  1606. * local APIC to INT and NMI lines.
  1607. */
  1608. apic_printk(APIC_VERBOSE, "leaving PIC mode, "
  1609. "enabling APIC mode.\n");
  1610. imcr_pic_to_apic();
  1611. }
  1612. #endif
  1613. if (apic->enable_apic_mode)
  1614. apic->enable_apic_mode();
  1615. }
  1616. /**
  1617. * disconnect_bsp_APIC - detach the APIC from the interrupt system
  1618. * @virt_wire_setup: indicates, whether virtual wire mode is selected
  1619. *
  1620. * Virtual wire mode is necessary to deliver legacy interrupts even when the
  1621. * APIC is disabled.
  1622. */
  1623. void disconnect_bsp_APIC(int virt_wire_setup)
  1624. {
  1625. unsigned int value;
  1626. #ifdef CONFIG_X86_32
  1627. if (pic_mode) {
  1628. /*
  1629. * Put the board back into PIC mode (has an effect only on
  1630. * certain older boards). Note that APIC interrupts, including
  1631. * IPIs, won't work beyond this point! The only exception are
  1632. * INIT IPIs.
  1633. */
  1634. apic_printk(APIC_VERBOSE, "disabling APIC mode, "
  1635. "entering PIC mode.\n");
  1636. imcr_apic_to_pic();
  1637. return;
  1638. }
  1639. #endif
  1640. /* Go back to Virtual Wire compatibility mode */
  1641. /* For the spurious interrupt use vector F, and enable it */
  1642. value = apic_read(APIC_SPIV);
  1643. value &= ~APIC_VECTOR_MASK;
  1644. value |= APIC_SPIV_APIC_ENABLED;
  1645. value |= 0xf;
  1646. apic_write(APIC_SPIV, value);
  1647. if (!virt_wire_setup) {
  1648. /*
  1649. * For LVT0 make it edge triggered, active high,
  1650. * external and enabled
  1651. */
  1652. value = apic_read(APIC_LVT0);
  1653. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1654. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1655. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1656. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1657. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
  1658. apic_write(APIC_LVT0, value);
  1659. } else {
  1660. /* Disable LVT0 */
  1661. apic_write(APIC_LVT0, APIC_LVT_MASKED);
  1662. }
  1663. /*
  1664. * For LVT1 make it edge triggered, active high,
  1665. * nmi and enabled
  1666. */
  1667. value = apic_read(APIC_LVT1);
  1668. value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
  1669. APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
  1670. APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
  1671. value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
  1672. value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
  1673. apic_write(APIC_LVT1, value);
  1674. }
  1675. void __cpuinit generic_processor_info(int apicid, int version)
  1676. {
  1677. int cpu, max = nr_cpu_ids;
  1678. bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
  1679. phys_cpu_present_map);
  1680. /*
  1681. * If boot cpu has not been detected yet, then only allow upto
  1682. * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
  1683. */
  1684. if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
  1685. apicid != boot_cpu_physical_apicid) {
  1686. int thiscpu = max + disabled_cpus - 1;
  1687. pr_warning(
  1688. "ACPI: NR_CPUS/possible_cpus limit of %i almost"
  1689. " reached. Keeping one slot for boot cpu."
  1690. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1691. disabled_cpus++;
  1692. return;
  1693. }
  1694. if (num_processors >= nr_cpu_ids) {
  1695. int thiscpu = max + disabled_cpus;
  1696. pr_warning(
  1697. "ACPI: NR_CPUS/possible_cpus limit of %i reached."
  1698. " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
  1699. disabled_cpus++;
  1700. return;
  1701. }
  1702. num_processors++;
  1703. if (apicid == boot_cpu_physical_apicid) {
  1704. /*
  1705. * x86_bios_cpu_apicid is required to have processors listed
  1706. * in same order as logical cpu numbers. Hence the first
  1707. * entry is BSP, and so on.
  1708. * boot_cpu_init() already hold bit 0 in cpu_present_mask
  1709. * for BSP.
  1710. */
  1711. cpu = 0;
  1712. } else
  1713. cpu = cpumask_next_zero(-1, cpu_present_mask);
  1714. /*
  1715. * Validate version
  1716. */
  1717. if (version == 0x0) {
  1718. pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
  1719. cpu, apicid);
  1720. version = 0x10;
  1721. }
  1722. apic_version[apicid] = version;
  1723. if (version != apic_version[boot_cpu_physical_apicid]) {
  1724. pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
  1725. apic_version[boot_cpu_physical_apicid], cpu, version);
  1726. }
  1727. physid_set(apicid, phys_cpu_present_map);
  1728. if (apicid > max_physical_apicid)
  1729. max_physical_apicid = apicid;
  1730. #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
  1731. early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
  1732. early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
  1733. #endif
  1734. #ifdef CONFIG_X86_32
  1735. early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
  1736. apic->x86_32_early_logical_apicid(cpu);
  1737. #endif
  1738. set_cpu_possible(cpu, true);
  1739. set_cpu_present(cpu, true);
  1740. }
  1741. int hard_smp_processor_id(void)
  1742. {
  1743. return read_apic_id();
  1744. }
  1745. void default_init_apic_ldr(void)
  1746. {
  1747. unsigned long val;
  1748. apic_write(APIC_DFR, APIC_DFR_VALUE);
  1749. val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
  1750. val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
  1751. apic_write(APIC_LDR, val);
  1752. }
  1753. /*
  1754. * Power management
  1755. */
  1756. #ifdef CONFIG_PM
  1757. static struct {
  1758. /*
  1759. * 'active' is true if the local APIC was enabled by us and
  1760. * not the BIOS; this signifies that we are also responsible
  1761. * for disabling it before entering apm/acpi suspend
  1762. */
  1763. int active;
  1764. /* r/w apic fields */
  1765. unsigned int apic_id;
  1766. unsigned int apic_taskpri;
  1767. unsigned int apic_ldr;
  1768. unsigned int apic_dfr;
  1769. unsigned int apic_spiv;
  1770. unsigned int apic_lvtt;
  1771. unsigned int apic_lvtpc;
  1772. unsigned int apic_lvt0;
  1773. unsigned int apic_lvt1;
  1774. unsigned int apic_lvterr;
  1775. unsigned int apic_tmict;
  1776. unsigned int apic_tdcr;
  1777. unsigned int apic_thmr;
  1778. } apic_pm_state;
  1779. static int lapic_suspend(void)
  1780. {
  1781. unsigned long flags;
  1782. int maxlvt;
  1783. if (!apic_pm_state.active)
  1784. return 0;
  1785. maxlvt = lapic_get_maxlvt();
  1786. apic_pm_state.apic_id = apic_read(APIC_ID);
  1787. apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
  1788. apic_pm_state.apic_ldr = apic_read(APIC_LDR);
  1789. apic_pm_state.apic_dfr = apic_read(APIC_DFR);
  1790. apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
  1791. apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
  1792. if (maxlvt >= 4)
  1793. apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
  1794. apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
  1795. apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
  1796. apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
  1797. apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
  1798. apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
  1799. #ifdef CONFIG_X86_THERMAL_VECTOR
  1800. if (maxlvt >= 5)
  1801. apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
  1802. #endif
  1803. local_irq_save(flags);
  1804. disable_local_APIC();
  1805. if (intr_remapping_enabled)
  1806. disable_intr_remapping();
  1807. local_irq_restore(flags);
  1808. return 0;
  1809. }
  1810. static void lapic_resume(void)
  1811. {
  1812. unsigned int l, h;
  1813. unsigned long flags;
  1814. int maxlvt;
  1815. if (!apic_pm_state.active)
  1816. return;
  1817. local_irq_save(flags);
  1818. if (intr_remapping_enabled) {
  1819. /*
  1820. * IO-APIC and PIC have their own resume routines.
  1821. * We just mask them here to make sure the interrupt
  1822. * subsystem is completely quiet while we enable x2apic
  1823. * and interrupt-remapping.
  1824. */
  1825. mask_ioapic_entries();
  1826. legacy_pic->mask_all();
  1827. }
  1828. if (x2apic_mode)
  1829. enable_x2apic();
  1830. else {
  1831. /*
  1832. * Make sure the APICBASE points to the right address
  1833. *
  1834. * FIXME! This will be wrong if we ever support suspend on
  1835. * SMP! We'll need to do this as part of the CPU restore!
  1836. */
  1837. rdmsr(MSR_IA32_APICBASE, l, h);
  1838. l &= ~MSR_IA32_APICBASE_BASE;
  1839. l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
  1840. wrmsr(MSR_IA32_APICBASE, l, h);
  1841. }
  1842. maxlvt = lapic_get_maxlvt();
  1843. apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
  1844. apic_write(APIC_ID, apic_pm_state.apic_id);
  1845. apic_write(APIC_DFR, apic_pm_state.apic_dfr);
  1846. apic_write(APIC_LDR, apic_pm_state.apic_ldr);
  1847. apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
  1848. apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
  1849. apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
  1850. apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
  1851. #if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
  1852. if (maxlvt >= 5)
  1853. apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
  1854. #endif
  1855. if (maxlvt >= 4)
  1856. apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
  1857. apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
  1858. apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
  1859. apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
  1860. apic_write(APIC_ESR, 0);
  1861. apic_read(APIC_ESR);
  1862. apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
  1863. apic_write(APIC_ESR, 0);
  1864. apic_read(APIC_ESR);
  1865. if (intr_remapping_enabled)
  1866. reenable_intr_remapping(x2apic_mode);
  1867. local_irq_restore(flags);
  1868. }
  1869. /*
  1870. * This device has no shutdown method - fully functioning local APICs
  1871. * are needed on every CPU up until machine_halt/restart/poweroff.
  1872. */
  1873. static struct syscore_ops lapic_syscore_ops = {
  1874. .resume = lapic_resume,
  1875. .suspend = lapic_suspend,
  1876. };
  1877. static void __cpuinit apic_pm_activate(void)
  1878. {
  1879. apic_pm_state.active = 1;
  1880. }
  1881. static int __init init_lapic_sysfs(void)
  1882. {
  1883. /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
  1884. if (cpu_has_apic)
  1885. register_syscore_ops(&lapic_syscore_ops);
  1886. return 0;
  1887. }
  1888. /* local apic needs to resume before other devices access its registers. */
  1889. core_initcall(init_lapic_sysfs);
  1890. #else /* CONFIG_PM */
  1891. static void apic_pm_activate(void) { }
  1892. #endif /* CONFIG_PM */
  1893. #ifdef CONFIG_X86_64
  1894. static int __cpuinit apic_cluster_num(void)
  1895. {
  1896. int i, clusters, zeros;
  1897. unsigned id;
  1898. u16 *bios_cpu_apicid;
  1899. DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);
  1900. bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
  1901. bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
  1902. for (i = 0; i < nr_cpu_ids; i++) {
  1903. /* are we being called early in kernel startup? */
  1904. if (bios_cpu_apicid) {
  1905. id = bios_cpu_apicid[i];
  1906. } else if (i < nr_cpu_ids) {
  1907. if (cpu_present(i))
  1908. id = per_cpu(x86_bios_cpu_apicid, i);
  1909. else
  1910. continue;
  1911. } else
  1912. break;
  1913. if (id != BAD_APICID)
  1914. __set_bit(APIC_CLUSTERID(id), clustermap);
  1915. }
  1916. /* Problem: Partially populated chassis may not have CPUs in some of
  1917. * the APIC clusters they have been allocated. Only present CPUs have
  1918. * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
  1919. * Since clusters are allocated sequentially, count zeros only if
  1920. * they are bounded by ones.
  1921. */
  1922. clusters = 0;
  1923. zeros = 0;
  1924. for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
  1925. if (test_bit(i, clustermap)) {
  1926. clusters += 1 + zeros;
  1927. zeros = 0;
  1928. } else
  1929. ++zeros;
  1930. }
  1931. return clusters;
  1932. }
  1933. static int __cpuinitdata multi_checked;
  1934. static int __cpuinitdata multi;
  1935. static int __cpuinit set_multi(const struct dmi_system_id *d)
  1936. {
  1937. if (multi)
  1938. return 0;
  1939. pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
  1940. multi = 1;
  1941. return 0;
  1942. }
  1943. static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = {
  1944. {
  1945. .callback = set_multi,
  1946. .ident = "IBM System Summit2",
  1947. .matches = {
  1948. DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
  1949. DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
  1950. },
  1951. },
  1952. {}
  1953. };
  1954. static void __cpuinit dmi_check_multi(void)
  1955. {
  1956. if (multi_checked)
  1957. return;
  1958. dmi_check_system(multi_dmi_table);
  1959. multi_checked = 1;
  1960. }
  1961. /*
  1962. * apic_is_clustered_box() -- Check if we can expect good TSC
  1963. *
  1964. * Thus far, the major user of this is IBM's Summit2 series:
  1965. * Clustered boxes may have unsynced TSC problems if they are
  1966. * multi-chassis.
  1967. * Use DMI to check them
  1968. */
  1969. __cpuinit int apic_is_clustered_box(void)
  1970. {
  1971. dmi_check_multi();
  1972. if (multi)
  1973. return 1;
  1974. if (!is_vsmp_box())
  1975. return 0;
  1976. /*
  1977. * ScaleMP vSMPowered boxes have one cluster per board and TSCs are
  1978. * not guaranteed to be synced between boards
  1979. */
  1980. if (apic_cluster_num() > 1)
  1981. return 1;
  1982. return 0;
  1983. }
  1984. #endif
  1985. /*
  1986. * APIC command line parameters
  1987. */
  1988. static int __init setup_disableapic(char *arg)
  1989. {
  1990. disable_apic = 1;
  1991. setup_clear_cpu_cap(X86_FEATURE_APIC);
  1992. return 0;
  1993. }
  1994. early_param("disableapic", setup_disableapic);
  1995. /* same as disableapic, for compatibility */
  1996. static int __init setup_nolapic(char *arg)
  1997. {
  1998. return setup_disableapic(arg);
  1999. }
  2000. early_param("nolapic", setup_nolapic);
  2001. static int __init parse_lapic_timer_c2_ok(char *arg)
  2002. {
  2003. local_apic_timer_c2_ok = 1;
  2004. return 0;
  2005. }
  2006. early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
  2007. static int __init parse_disable_apic_timer(char *arg)
  2008. {
  2009. disable_apic_timer = 1;
  2010. return 0;
  2011. }
  2012. early_param("noapictimer", parse_disable_apic_timer);
  2013. static int __init parse_nolapic_timer(char *arg)
  2014. {
  2015. disable_apic_timer = 1;
  2016. return 0;
  2017. }
  2018. early_param("nolapic_timer", parse_nolapic_timer);
  2019. static int __init apic_set_verbosity(char *arg)
  2020. {
  2021. if (!arg) {
  2022. #ifdef CONFIG_X86_64
  2023. skip_ioapic_setup = 0;
  2024. return 0;
  2025. #endif
  2026. return -EINVAL;
  2027. }
  2028. if (strcmp("debug", arg) == 0)
  2029. apic_verbosity = APIC_DEBUG;
  2030. else if (strcmp("verbose", arg) == 0)
  2031. apic_verbosity = APIC_VERBOSE;
  2032. else {
  2033. pr_warning("APIC Verbosity level %s not recognised"
  2034. " use apic=verbose or apic=debug\n", arg);
  2035. return -EINVAL;
  2036. }
  2037. return 0;
  2038. }
  2039. early_param("apic", apic_set_verbosity);
  2040. static int __init lapic_insert_resource(void)
  2041. {
  2042. if (!apic_phys)
  2043. return -1;
  2044. /* Put local APIC into the resource map. */
  2045. lapic_resource.start = apic_phys;
  2046. lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
  2047. insert_resource(&iomem_resource, &lapic_resource);
  2048. return 0;
  2049. }
  2050. /*
  2051. * need call insert after e820_reserve_resources()
  2052. * that is using request_resource
  2053. */
  2054. late_initcall(lapic_insert_resource);