io_apic.c 98 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170
  1. /*
  2. * Intel IO-APIC support for multi-Pentium hosts.
  3. *
  4. * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
  5. *
  6. * Many thanks to Stig Venaas for trying out countless experimental
  7. * patches and reporting/debugging problems patiently!
  8. *
  9. * (c) 1999, Multiple IO-APIC support, developed by
  10. * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
  11. * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
  12. * further tested and cleaned up by Zach Brown <zab@redhat.com>
  13. * and Ingo Molnar <mingo@redhat.com>
  14. *
  15. * Fixes
  16. * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
  17. * thanks to Eric Gilmore
  18. * and Rolf G. Tews
  19. * for testing these extensively
  20. * Paul Diefenbaugh : Added full ACPI support
  21. */
  22. #include <linux/mm.h>
  23. #include <linux/interrupt.h>
  24. #include <linux/init.h>
  25. #include <linux/delay.h>
  26. #include <linux/sched.h>
  27. #include <linux/pci.h>
  28. #include <linux/mc146818rtc.h>
  29. #include <linux/compiler.h>
  30. #include <linux/acpi.h>
  31. #include <linux/module.h>
  32. #include <linux/sysdev.h>
  33. #include <linux/msi.h>
  34. #include <linux/htirq.h>
  35. #include <linux/freezer.h>
  36. #include <linux/kthread.h>
  37. #include <linux/jiffies.h> /* time_after() */
  38. #ifdef CONFIG_ACPI
  39. #include <acpi/acpi_bus.h>
  40. #endif
  41. #include <linux/bootmem.h>
  42. #include <linux/dmar.h>
  43. #include <linux/hpet.h>
  44. #include <asm/idle.h>
  45. #include <asm/io.h>
  46. #include <asm/smp.h>
  47. #include <asm/desc.h>
  48. #include <asm/proto.h>
  49. #include <asm/acpi.h>
  50. #include <asm/dma.h>
  51. #include <asm/timer.h>
  52. #include <asm/i8259.h>
  53. #include <asm/nmi.h>
  54. #include <asm/msidef.h>
  55. #include <asm/hypertransport.h>
  56. #include <asm/setup.h>
  57. #include <asm/irq_remapping.h>
  58. #include <asm/hpet.h>
  59. #include <asm/uv/uv_hub.h>
  60. #include <asm/uv/uv_irq.h>
  61. #include <mach_ipi.h>
  62. #include <mach_apic.h>
  63. #include <mach_apicdef.h>
  64. #define __apicdebuginit(type) static type __init
  65. /*
  66. * Is the SiS APIC rmw bug present ?
  67. * -1 = don't know, 0 = no, 1 = yes
  68. */
  69. int sis_apic_bug = -1;
  70. static DEFINE_SPINLOCK(ioapic_lock);
  71. static DEFINE_SPINLOCK(vector_lock);
  72. /*
  73. * # of IRQ routing registers
  74. */
  75. int nr_ioapic_registers[MAX_IO_APICS];
  76. /* I/O APIC entries */
  77. struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
  78. int nr_ioapics;
  79. /* MP IRQ source entries */
  80. struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
  81. /* # of MP IRQ source entries */
  82. int mp_irq_entries;
  83. #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
  84. int mp_bus_id_to_type[MAX_MP_BUSSES];
  85. #endif
  86. DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
  87. int skip_ioapic_setup;
  88. static int __init parse_noapic(char *str)
  89. {
  90. /* disable IO-APIC */
  91. disable_ioapic_setup();
  92. return 0;
  93. }
  94. early_param("noapic", parse_noapic);
  95. struct irq_pin_list;
  96. /*
  97. * This is performance-critical, we want to do it O(1)
  98. *
  99. * the indexing order of this array favors 1:1 mappings
  100. * between pins and IRQs.
  101. */
  102. struct irq_pin_list {
  103. int apic, pin;
  104. struct irq_pin_list *next;
  105. };
  106. static struct irq_pin_list *get_one_free_irq_2_pin(int cpu)
  107. {
  108. struct irq_pin_list *pin;
  109. int node;
  110. node = cpu_to_node(cpu);
  111. pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
  112. printk(KERN_DEBUG " alloc irq_2_pin on cpu %d node %d\n", cpu, node);
  113. return pin;
  114. }
  115. struct irq_cfg {
  116. struct irq_pin_list *irq_2_pin;
  117. cpumask_var_t domain;
  118. cpumask_var_t old_domain;
  119. unsigned move_cleanup_count;
  120. u8 vector;
  121. u8 move_in_progress : 1;
  122. #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
  123. u8 move_desc_pending : 1;
  124. #endif
  125. };
  126. /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
  127. #ifdef CONFIG_SPARSE_IRQ
  128. static struct irq_cfg irq_cfgx[] = {
  129. #else
  130. static struct irq_cfg irq_cfgx[NR_IRQS] = {
  131. #endif
  132. [0] = { .vector = IRQ0_VECTOR, },
  133. [1] = { .vector = IRQ1_VECTOR, },
  134. [2] = { .vector = IRQ2_VECTOR, },
  135. [3] = { .vector = IRQ3_VECTOR, },
  136. [4] = { .vector = IRQ4_VECTOR, },
  137. [5] = { .vector = IRQ5_VECTOR, },
  138. [6] = { .vector = IRQ6_VECTOR, },
  139. [7] = { .vector = IRQ7_VECTOR, },
  140. [8] = { .vector = IRQ8_VECTOR, },
  141. [9] = { .vector = IRQ9_VECTOR, },
  142. [10] = { .vector = IRQ10_VECTOR, },
  143. [11] = { .vector = IRQ11_VECTOR, },
  144. [12] = { .vector = IRQ12_VECTOR, },
  145. [13] = { .vector = IRQ13_VECTOR, },
  146. [14] = { .vector = IRQ14_VECTOR, },
  147. [15] = { .vector = IRQ15_VECTOR, },
  148. };
  149. void __init arch_early_irq_init(void)
  150. {
  151. struct irq_cfg *cfg;
  152. struct irq_desc *desc;
  153. int count;
  154. int i;
  155. cfg = irq_cfgx;
  156. count = ARRAY_SIZE(irq_cfgx);
  157. for (i = 0; i < count; i++) {
  158. desc = irq_to_desc(i);
  159. desc->chip_data = &cfg[i];
  160. alloc_bootmem_cpumask_var(&cfg[i].domain);
  161. alloc_bootmem_cpumask_var(&cfg[i].old_domain);
  162. if (i < NR_IRQS_LEGACY)
  163. cpumask_setall(cfg[i].domain);
  164. }
  165. }
  166. #ifdef CONFIG_SPARSE_IRQ
  167. static struct irq_cfg *irq_cfg(unsigned int irq)
  168. {
  169. struct irq_cfg *cfg = NULL;
  170. struct irq_desc *desc;
  171. desc = irq_to_desc(irq);
  172. if (desc)
  173. cfg = desc->chip_data;
  174. return cfg;
  175. }
  176. static struct irq_cfg *get_one_free_irq_cfg(int cpu)
  177. {
  178. struct irq_cfg *cfg;
  179. int node;
  180. node = cpu_to_node(cpu);
  181. cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
  182. if (cfg) {
  183. /* FIXME: needs alloc_cpumask_var_node() */
  184. if (!alloc_cpumask_var(&cfg->domain, GFP_ATOMIC)) {
  185. kfree(cfg);
  186. cfg = NULL;
  187. } else if (!alloc_cpumask_var(&cfg->old_domain, GFP_ATOMIC)) {
  188. free_cpumask_var(cfg->domain);
  189. kfree(cfg);
  190. cfg = NULL;
  191. } else {
  192. cpumask_clear(cfg->domain);
  193. cpumask_clear(cfg->old_domain);
  194. }
  195. }
  196. printk(KERN_DEBUG " alloc irq_cfg on cpu %d node %d\n", cpu, node);
  197. return cfg;
  198. }
  199. void arch_init_chip_data(struct irq_desc *desc, int cpu)
  200. {
  201. struct irq_cfg *cfg;
  202. cfg = desc->chip_data;
  203. if (!cfg) {
  204. desc->chip_data = get_one_free_irq_cfg(cpu);
  205. if (!desc->chip_data) {
  206. printk(KERN_ERR "can not alloc irq_cfg\n");
  207. BUG_ON(1);
  208. }
  209. }
  210. }
  211. #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
  212. static void
  213. init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
  214. {
  215. struct irq_pin_list *old_entry, *head, *tail, *entry;
  216. cfg->irq_2_pin = NULL;
  217. old_entry = old_cfg->irq_2_pin;
  218. if (!old_entry)
  219. return;
  220. entry = get_one_free_irq_2_pin(cpu);
  221. if (!entry)
  222. return;
  223. entry->apic = old_entry->apic;
  224. entry->pin = old_entry->pin;
  225. head = entry;
  226. tail = entry;
  227. old_entry = old_entry->next;
  228. while (old_entry) {
  229. entry = get_one_free_irq_2_pin(cpu);
  230. if (!entry) {
  231. entry = head;
  232. while (entry) {
  233. head = entry->next;
  234. kfree(entry);
  235. entry = head;
  236. }
  237. /* still use the old one */
  238. return;
  239. }
  240. entry->apic = old_entry->apic;
  241. entry->pin = old_entry->pin;
  242. tail->next = entry;
  243. tail = entry;
  244. old_entry = old_entry->next;
  245. }
  246. tail->next = NULL;
  247. cfg->irq_2_pin = head;
  248. }
  249. static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
  250. {
  251. struct irq_pin_list *entry, *next;
  252. if (old_cfg->irq_2_pin == cfg->irq_2_pin)
  253. return;
  254. entry = old_cfg->irq_2_pin;
  255. while (entry) {
  256. next = entry->next;
  257. kfree(entry);
  258. entry = next;
  259. }
  260. old_cfg->irq_2_pin = NULL;
  261. }
  262. void arch_init_copy_chip_data(struct irq_desc *old_desc,
  263. struct irq_desc *desc, int cpu)
  264. {
  265. struct irq_cfg *cfg;
  266. struct irq_cfg *old_cfg;
  267. cfg = get_one_free_irq_cfg(cpu);
  268. if (!cfg)
  269. return;
  270. desc->chip_data = cfg;
  271. old_cfg = old_desc->chip_data;
  272. memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
  273. init_copy_irq_2_pin(old_cfg, cfg, cpu);
  274. }
  275. static void free_irq_cfg(struct irq_cfg *old_cfg)
  276. {
  277. kfree(old_cfg);
  278. }
  279. void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
  280. {
  281. struct irq_cfg *old_cfg, *cfg;
  282. old_cfg = old_desc->chip_data;
  283. cfg = desc->chip_data;
  284. if (old_cfg == cfg)
  285. return;
  286. if (old_cfg) {
  287. free_irq_2_pin(old_cfg, cfg);
  288. free_irq_cfg(old_cfg);
  289. old_desc->chip_data = NULL;
  290. }
  291. }
  292. static void
  293. set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
  294. {
  295. struct irq_cfg *cfg = desc->chip_data;
  296. if (!cfg->move_in_progress) {
  297. /* it means that domain is not changed */
  298. if (!cpumask_intersects(&desc->affinity, mask))
  299. cfg->move_desc_pending = 1;
  300. }
  301. }
  302. #endif
  303. #else
  304. static struct irq_cfg *irq_cfg(unsigned int irq)
  305. {
  306. return irq < nr_irqs ? irq_cfgx + irq : NULL;
  307. }
  308. #endif
  309. #ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC
  310. static inline void
  311. set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
  312. {
  313. }
  314. #endif
  315. struct io_apic {
  316. unsigned int index;
  317. unsigned int unused[3];
  318. unsigned int data;
  319. };
  320. static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
  321. {
  322. return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
  323. + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
  324. }
  325. static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
  326. {
  327. struct io_apic __iomem *io_apic = io_apic_base(apic);
  328. writel(reg, &io_apic->index);
  329. return readl(&io_apic->data);
  330. }
  331. static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
  332. {
  333. struct io_apic __iomem *io_apic = io_apic_base(apic);
  334. writel(reg, &io_apic->index);
  335. writel(value, &io_apic->data);
  336. }
  337. /*
  338. * Re-write a value: to be used for read-modify-write
  339. * cycles where the read already set up the index register.
  340. *
  341. * Older SiS APIC requires we rewrite the index register
  342. */
  343. static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
  344. {
  345. struct io_apic __iomem *io_apic = io_apic_base(apic);
  346. if (sis_apic_bug)
  347. writel(reg, &io_apic->index);
  348. writel(value, &io_apic->data);
  349. }
  350. static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
  351. {
  352. struct irq_pin_list *entry;
  353. unsigned long flags;
  354. spin_lock_irqsave(&ioapic_lock, flags);
  355. entry = cfg->irq_2_pin;
  356. for (;;) {
  357. unsigned int reg;
  358. int pin;
  359. if (!entry)
  360. break;
  361. pin = entry->pin;
  362. reg = io_apic_read(entry->apic, 0x10 + pin*2);
  363. /* Is the remote IRR bit set? */
  364. if (reg & IO_APIC_REDIR_REMOTE_IRR) {
  365. spin_unlock_irqrestore(&ioapic_lock, flags);
  366. return true;
  367. }
  368. if (!entry->next)
  369. break;
  370. entry = entry->next;
  371. }
  372. spin_unlock_irqrestore(&ioapic_lock, flags);
  373. return false;
  374. }
  375. union entry_union {
  376. struct { u32 w1, w2; };
  377. struct IO_APIC_route_entry entry;
  378. };
  379. static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
  380. {
  381. union entry_union eu;
  382. unsigned long flags;
  383. spin_lock_irqsave(&ioapic_lock, flags);
  384. eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
  385. eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
  386. spin_unlock_irqrestore(&ioapic_lock, flags);
  387. return eu.entry;
  388. }
  389. /*
  390. * When we write a new IO APIC routing entry, we need to write the high
  391. * word first! If the mask bit in the low word is clear, we will enable
  392. * the interrupt, and we need to make sure the entry is fully populated
  393. * before that happens.
  394. */
  395. static void
  396. __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
  397. {
  398. union entry_union eu;
  399. eu.entry = e;
  400. io_apic_write(apic, 0x11 + 2*pin, eu.w2);
  401. io_apic_write(apic, 0x10 + 2*pin, eu.w1);
  402. }
  403. static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
  404. {
  405. unsigned long flags;
  406. spin_lock_irqsave(&ioapic_lock, flags);
  407. __ioapic_write_entry(apic, pin, e);
  408. spin_unlock_irqrestore(&ioapic_lock, flags);
  409. }
  410. /*
  411. * When we mask an IO APIC routing entry, we need to write the low
  412. * word first, in order to set the mask bit before we change the
  413. * high bits!
  414. */
  415. static void ioapic_mask_entry(int apic, int pin)
  416. {
  417. unsigned long flags;
  418. union entry_union eu = { .entry.mask = 1 };
  419. spin_lock_irqsave(&ioapic_lock, flags);
  420. io_apic_write(apic, 0x10 + 2*pin, eu.w1);
  421. io_apic_write(apic, 0x11 + 2*pin, eu.w2);
  422. spin_unlock_irqrestore(&ioapic_lock, flags);
  423. }
  424. #ifdef CONFIG_SMP
  425. static void send_cleanup_vector(struct irq_cfg *cfg)
  426. {
  427. cpumask_var_t cleanup_mask;
  428. if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
  429. unsigned int i;
  430. cfg->move_cleanup_count = 0;
  431. for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
  432. cfg->move_cleanup_count++;
  433. for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
  434. send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
  435. } else {
  436. cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
  437. cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
  438. send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
  439. free_cpumask_var(cleanup_mask);
  440. }
  441. cfg->move_in_progress = 0;
  442. }
  443. static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
  444. {
  445. int apic, pin;
  446. struct irq_pin_list *entry;
  447. u8 vector = cfg->vector;
  448. entry = cfg->irq_2_pin;
  449. for (;;) {
  450. unsigned int reg;
  451. if (!entry)
  452. break;
  453. apic = entry->apic;
  454. pin = entry->pin;
  455. #ifdef CONFIG_INTR_REMAP
  456. /*
  457. * With interrupt-remapping, destination information comes
  458. * from interrupt-remapping table entry.
  459. */
  460. if (!irq_remapped(irq))
  461. io_apic_write(apic, 0x11 + pin*2, dest);
  462. #else
  463. io_apic_write(apic, 0x11 + pin*2, dest);
  464. #endif
  465. reg = io_apic_read(apic, 0x10 + pin*2);
  466. reg &= ~IO_APIC_REDIR_VECTOR_MASK;
  467. reg |= vector;
  468. io_apic_modify(apic, 0x10 + pin*2, reg);
  469. if (!entry->next)
  470. break;
  471. entry = entry->next;
  472. }
  473. }
  474. static int
  475. assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
  476. /*
  477. * Either sets desc->affinity to a valid value, and returns cpu_mask_to_apicid
  478. * of that, or returns BAD_APICID and leaves desc->affinity untouched.
  479. */
  480. static unsigned int
  481. set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
  482. {
  483. struct irq_cfg *cfg;
  484. unsigned int irq;
  485. if (!cpumask_intersects(mask, cpu_online_mask))
  486. return BAD_APICID;
  487. irq = desc->irq;
  488. cfg = desc->chip_data;
  489. if (assign_irq_vector(irq, cfg, mask))
  490. return BAD_APICID;
  491. cpumask_and(&desc->affinity, cfg->domain, mask);
  492. set_extra_move_desc(desc, mask);
  493. return cpu_mask_to_apicid_and(&desc->affinity, cpu_online_mask);
  494. }
  495. static void
  496. set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
  497. {
  498. struct irq_cfg *cfg;
  499. unsigned long flags;
  500. unsigned int dest;
  501. unsigned int irq;
  502. irq = desc->irq;
  503. cfg = desc->chip_data;
  504. spin_lock_irqsave(&ioapic_lock, flags);
  505. dest = set_desc_affinity(desc, mask);
  506. if (dest != BAD_APICID) {
  507. /* Only the high 8 bits are valid. */
  508. dest = SET_APIC_LOGICAL_ID(dest);
  509. __target_IO_APIC_irq(irq, dest, cfg);
  510. }
  511. spin_unlock_irqrestore(&ioapic_lock, flags);
  512. }
  513. static void
  514. set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
  515. {
  516. struct irq_desc *desc;
  517. desc = irq_to_desc(irq);
  518. set_ioapic_affinity_irq_desc(desc, mask);
  519. }
  520. #endif /* CONFIG_SMP */
  521. /*
  522. * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
  523. * shared ISA-space IRQs, so we have to support them. We are super
  524. * fast in the common case, and fast for shared ISA-space IRQs.
  525. */
  526. static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
  527. {
  528. struct irq_pin_list *entry;
  529. entry = cfg->irq_2_pin;
  530. if (!entry) {
  531. entry = get_one_free_irq_2_pin(cpu);
  532. if (!entry) {
  533. printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
  534. apic, pin);
  535. return;
  536. }
  537. cfg->irq_2_pin = entry;
  538. entry->apic = apic;
  539. entry->pin = pin;
  540. return;
  541. }
  542. while (entry->next) {
  543. /* not again, please */
  544. if (entry->apic == apic && entry->pin == pin)
  545. return;
  546. entry = entry->next;
  547. }
  548. entry->next = get_one_free_irq_2_pin(cpu);
  549. entry = entry->next;
  550. entry->apic = apic;
  551. entry->pin = pin;
  552. }
  553. /*
  554. * Reroute an IRQ to a different pin.
  555. */
  556. static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
  557. int oldapic, int oldpin,
  558. int newapic, int newpin)
  559. {
  560. struct irq_pin_list *entry = cfg->irq_2_pin;
  561. int replaced = 0;
  562. while (entry) {
  563. if (entry->apic == oldapic && entry->pin == oldpin) {
  564. entry->apic = newapic;
  565. entry->pin = newpin;
  566. replaced = 1;
  567. /* every one is different, right? */
  568. break;
  569. }
  570. entry = entry->next;
  571. }
  572. /* why? call replace before add? */
  573. if (!replaced)
  574. add_pin_to_irq_cpu(cfg, cpu, newapic, newpin);
  575. }
  576. static inline void io_apic_modify_irq(struct irq_cfg *cfg,
  577. int mask_and, int mask_or,
  578. void (*final)(struct irq_pin_list *entry))
  579. {
  580. int pin;
  581. struct irq_pin_list *entry;
  582. for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
  583. unsigned int reg;
  584. pin = entry->pin;
  585. reg = io_apic_read(entry->apic, 0x10 + pin * 2);
  586. reg &= mask_and;
  587. reg |= mask_or;
  588. io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
  589. if (final)
  590. final(entry);
  591. }
  592. }
  593. static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
  594. {
  595. io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
  596. }
  597. #ifdef CONFIG_X86_64
  598. void io_apic_sync(struct irq_pin_list *entry)
  599. {
  600. /*
  601. * Synchronize the IO-APIC and the CPU by doing
  602. * a dummy read from the IO-APIC
  603. */
  604. struct io_apic __iomem *io_apic;
  605. io_apic = io_apic_base(entry->apic);
  606. readl(&io_apic->data);
  607. }
  608. static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
  609. {
  610. io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
  611. }
  612. #else /* CONFIG_X86_32 */
  613. static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
  614. {
  615. io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL);
  616. }
  617. static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
  618. {
  619. io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
  620. IO_APIC_REDIR_MASKED, NULL);
  621. }
  622. static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
  623. {
  624. io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
  625. IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
  626. }
  627. #endif /* CONFIG_X86_32 */
  628. static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
  629. {
  630. struct irq_cfg *cfg = desc->chip_data;
  631. unsigned long flags;
  632. BUG_ON(!cfg);
  633. spin_lock_irqsave(&ioapic_lock, flags);
  634. __mask_IO_APIC_irq(cfg);
  635. spin_unlock_irqrestore(&ioapic_lock, flags);
  636. }
  637. static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
  638. {
  639. struct irq_cfg *cfg = desc->chip_data;
  640. unsigned long flags;
  641. spin_lock_irqsave(&ioapic_lock, flags);
  642. __unmask_IO_APIC_irq(cfg);
  643. spin_unlock_irqrestore(&ioapic_lock, flags);
  644. }
  645. static void mask_IO_APIC_irq(unsigned int irq)
  646. {
  647. struct irq_desc *desc = irq_to_desc(irq);
  648. mask_IO_APIC_irq_desc(desc);
  649. }
  650. static void unmask_IO_APIC_irq(unsigned int irq)
  651. {
  652. struct irq_desc *desc = irq_to_desc(irq);
  653. unmask_IO_APIC_irq_desc(desc);
  654. }
  655. static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
  656. {
  657. struct IO_APIC_route_entry entry;
  658. /* Check delivery_mode to be sure we're not clearing an SMI pin */
  659. entry = ioapic_read_entry(apic, pin);
  660. if (entry.delivery_mode == dest_SMI)
  661. return;
  662. /*
  663. * Disable it in the IO-APIC irq-routing table:
  664. */
  665. ioapic_mask_entry(apic, pin);
  666. }
  667. static void clear_IO_APIC (void)
  668. {
  669. int apic, pin;
  670. for (apic = 0; apic < nr_ioapics; apic++)
  671. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
  672. clear_IO_APIC_pin(apic, pin);
  673. }
  674. #if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
  675. void send_IPI_self(int vector)
  676. {
  677. unsigned int cfg;
  678. /*
  679. * Wait for idle.
  680. */
  681. apic_wait_icr_idle();
  682. cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
  683. /*
  684. * Send the IPI. The write to APIC_ICR fires this off.
  685. */
  686. apic_write(APIC_ICR, cfg);
  687. }
  688. #endif /* !CONFIG_SMP && CONFIG_X86_32*/
  689. #ifdef CONFIG_X86_32
  690. /*
  691. * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
  692. * specific CPU-side IRQs.
  693. */
  694. #define MAX_PIRQS 8
  695. static int pirq_entries [MAX_PIRQS];
  696. static int pirqs_enabled;
  697. static int __init ioapic_pirq_setup(char *str)
  698. {
  699. int i, max;
  700. int ints[MAX_PIRQS+1];
  701. get_options(str, ARRAY_SIZE(ints), ints);
  702. for (i = 0; i < MAX_PIRQS; i++)
  703. pirq_entries[i] = -1;
  704. pirqs_enabled = 1;
  705. apic_printk(APIC_VERBOSE, KERN_INFO
  706. "PIRQ redirection, working around broken MP-BIOS.\n");
  707. max = MAX_PIRQS;
  708. if (ints[0] < MAX_PIRQS)
  709. max = ints[0];
  710. for (i = 0; i < max; i++) {
  711. apic_printk(APIC_VERBOSE, KERN_DEBUG
  712. "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
  713. /*
  714. * PIRQs are mapped upside down, usually.
  715. */
  716. pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
  717. }
  718. return 1;
  719. }
  720. __setup("pirq=", ioapic_pirq_setup);
  721. #endif /* CONFIG_X86_32 */
  722. #ifdef CONFIG_INTR_REMAP
  723. /* I/O APIC RTE contents at the OS boot up */
  724. static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
  725. /*
  726. * Saves and masks all the unmasked IO-APIC RTE's
  727. */
  728. int save_mask_IO_APIC_setup(void)
  729. {
  730. union IO_APIC_reg_01 reg_01;
  731. unsigned long flags;
  732. int apic, pin;
  733. /*
  734. * The number of IO-APIC IRQ registers (== #pins):
  735. */
  736. for (apic = 0; apic < nr_ioapics; apic++) {
  737. spin_lock_irqsave(&ioapic_lock, flags);
  738. reg_01.raw = io_apic_read(apic, 1);
  739. spin_unlock_irqrestore(&ioapic_lock, flags);
  740. nr_ioapic_registers[apic] = reg_01.bits.entries+1;
  741. }
  742. for (apic = 0; apic < nr_ioapics; apic++) {
  743. early_ioapic_entries[apic] =
  744. kzalloc(sizeof(struct IO_APIC_route_entry) *
  745. nr_ioapic_registers[apic], GFP_KERNEL);
  746. if (!early_ioapic_entries[apic])
  747. goto nomem;
  748. }
  749. for (apic = 0; apic < nr_ioapics; apic++)
  750. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
  751. struct IO_APIC_route_entry entry;
  752. entry = early_ioapic_entries[apic][pin] =
  753. ioapic_read_entry(apic, pin);
  754. if (!entry.mask) {
  755. entry.mask = 1;
  756. ioapic_write_entry(apic, pin, entry);
  757. }
  758. }
  759. return 0;
  760. nomem:
  761. while (apic >= 0)
  762. kfree(early_ioapic_entries[apic--]);
  763. memset(early_ioapic_entries, 0,
  764. ARRAY_SIZE(early_ioapic_entries));
  765. return -ENOMEM;
  766. }
  767. void restore_IO_APIC_setup(void)
  768. {
  769. int apic, pin;
  770. for (apic = 0; apic < nr_ioapics; apic++) {
  771. if (!early_ioapic_entries[apic])
  772. break;
  773. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
  774. ioapic_write_entry(apic, pin,
  775. early_ioapic_entries[apic][pin]);
  776. kfree(early_ioapic_entries[apic]);
  777. early_ioapic_entries[apic] = NULL;
  778. }
  779. }
  780. void reinit_intr_remapped_IO_APIC(int intr_remapping)
  781. {
  782. /*
  783. * for now plain restore of previous settings.
  784. * TBD: In the case of OS enabling interrupt-remapping,
  785. * IO-APIC RTE's need to be setup to point to interrupt-remapping
  786. * table entries. for now, do a plain restore, and wait for
  787. * the setup_IO_APIC_irqs() to do proper initialization.
  788. */
  789. restore_IO_APIC_setup();
  790. }
  791. #endif
  792. /*
  793. * Find the IRQ entry number of a certain pin.
  794. */
  795. static int find_irq_entry(int apic, int pin, int type)
  796. {
  797. int i;
  798. for (i = 0; i < mp_irq_entries; i++)
  799. if (mp_irqs[i].mp_irqtype == type &&
  800. (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
  801. mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
  802. mp_irqs[i].mp_dstirq == pin)
  803. return i;
  804. return -1;
  805. }
  806. /*
  807. * Find the pin to which IRQ[irq] (ISA) is connected
  808. */
  809. static int __init find_isa_irq_pin(int irq, int type)
  810. {
  811. int i;
  812. for (i = 0; i < mp_irq_entries; i++) {
  813. int lbus = mp_irqs[i].mp_srcbus;
  814. if (test_bit(lbus, mp_bus_not_pci) &&
  815. (mp_irqs[i].mp_irqtype == type) &&
  816. (mp_irqs[i].mp_srcbusirq == irq))
  817. return mp_irqs[i].mp_dstirq;
  818. }
  819. return -1;
  820. }
  821. static int __init find_isa_irq_apic(int irq, int type)
  822. {
  823. int i;
  824. for (i = 0; i < mp_irq_entries; i++) {
  825. int lbus = mp_irqs[i].mp_srcbus;
  826. if (test_bit(lbus, mp_bus_not_pci) &&
  827. (mp_irqs[i].mp_irqtype == type) &&
  828. (mp_irqs[i].mp_srcbusirq == irq))
  829. break;
  830. }
  831. if (i < mp_irq_entries) {
  832. int apic;
  833. for(apic = 0; apic < nr_ioapics; apic++) {
  834. if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
  835. return apic;
  836. }
  837. }
  838. return -1;
  839. }
  840. /*
  841. * Find a specific PCI IRQ entry.
  842. * Not an __init, possibly needed by modules
  843. */
  844. static int pin_2_irq(int idx, int apic, int pin);
  845. int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
  846. {
  847. int apic, i, best_guess = -1;
  848. apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
  849. bus, slot, pin);
  850. if (test_bit(bus, mp_bus_not_pci)) {
  851. apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
  852. return -1;
  853. }
  854. for (i = 0; i < mp_irq_entries; i++) {
  855. int lbus = mp_irqs[i].mp_srcbus;
  856. for (apic = 0; apic < nr_ioapics; apic++)
  857. if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
  858. mp_irqs[i].mp_dstapic == MP_APIC_ALL)
  859. break;
  860. if (!test_bit(lbus, mp_bus_not_pci) &&
  861. !mp_irqs[i].mp_irqtype &&
  862. (bus == lbus) &&
  863. (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
  864. int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
  865. if (!(apic || IO_APIC_IRQ(irq)))
  866. continue;
  867. if (pin == (mp_irqs[i].mp_srcbusirq & 3))
  868. return irq;
  869. /*
  870. * Use the first all-but-pin matching entry as a
  871. * best-guess fuzzy result for broken mptables.
  872. */
  873. if (best_guess < 0)
  874. best_guess = irq;
  875. }
  876. }
  877. return best_guess;
  878. }
  879. EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
  880. #if defined(CONFIG_EISA) || defined(CONFIG_MCA)
  881. /*
  882. * EISA Edge/Level control register, ELCR
  883. */
  884. static int EISA_ELCR(unsigned int irq)
  885. {
  886. if (irq < NR_IRQS_LEGACY) {
  887. unsigned int port = 0x4d0 + (irq >> 3);
  888. return (inb(port) >> (irq & 7)) & 1;
  889. }
  890. apic_printk(APIC_VERBOSE, KERN_INFO
  891. "Broken MPtable reports ISA irq %d\n", irq);
  892. return 0;
  893. }
  894. #endif
  895. /* ISA interrupts are always polarity zero edge triggered,
  896. * when listed as conforming in the MP table. */
  897. #define default_ISA_trigger(idx) (0)
  898. #define default_ISA_polarity(idx) (0)
  899. /* EISA interrupts are always polarity zero and can be edge or level
  900. * trigger depending on the ELCR value. If an interrupt is listed as
  901. * EISA conforming in the MP table, that means its trigger type must
  902. * be read in from the ELCR */
  903. #define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
  904. #define default_EISA_polarity(idx) default_ISA_polarity(idx)
  905. /* PCI interrupts are always polarity one level triggered,
  906. * when listed as conforming in the MP table. */
  907. #define default_PCI_trigger(idx) (1)
  908. #define default_PCI_polarity(idx) (1)
  909. /* MCA interrupts are always polarity zero level triggered,
  910. * when listed as conforming in the MP table. */
  911. #define default_MCA_trigger(idx) (1)
  912. #define default_MCA_polarity(idx) default_ISA_polarity(idx)
  913. static int MPBIOS_polarity(int idx)
  914. {
  915. int bus = mp_irqs[idx].mp_srcbus;
  916. int polarity;
  917. /*
  918. * Determine IRQ line polarity (high active or low active):
  919. */
  920. switch (mp_irqs[idx].mp_irqflag & 3)
  921. {
  922. case 0: /* conforms, ie. bus-type dependent polarity */
  923. if (test_bit(bus, mp_bus_not_pci))
  924. polarity = default_ISA_polarity(idx);
  925. else
  926. polarity = default_PCI_polarity(idx);
  927. break;
  928. case 1: /* high active */
  929. {
  930. polarity = 0;
  931. break;
  932. }
  933. case 2: /* reserved */
  934. {
  935. printk(KERN_WARNING "broken BIOS!!\n");
  936. polarity = 1;
  937. break;
  938. }
  939. case 3: /* low active */
  940. {
  941. polarity = 1;
  942. break;
  943. }
  944. default: /* invalid */
  945. {
  946. printk(KERN_WARNING "broken BIOS!!\n");
  947. polarity = 1;
  948. break;
  949. }
  950. }
  951. return polarity;
  952. }
  953. static int MPBIOS_trigger(int idx)
  954. {
  955. int bus = mp_irqs[idx].mp_srcbus;
  956. int trigger;
  957. /*
  958. * Determine IRQ trigger mode (edge or level sensitive):
  959. */
  960. switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
  961. {
  962. case 0: /* conforms, ie. bus-type dependent */
  963. if (test_bit(bus, mp_bus_not_pci))
  964. trigger = default_ISA_trigger(idx);
  965. else
  966. trigger = default_PCI_trigger(idx);
  967. #if defined(CONFIG_EISA) || defined(CONFIG_MCA)
  968. switch (mp_bus_id_to_type[bus]) {
  969. case MP_BUS_ISA: /* ISA pin */
  970. {
  971. /* set before the switch */
  972. break;
  973. }
  974. case MP_BUS_EISA: /* EISA pin */
  975. {
  976. trigger = default_EISA_trigger(idx);
  977. break;
  978. }
  979. case MP_BUS_PCI: /* PCI pin */
  980. {
  981. /* set before the switch */
  982. break;
  983. }
  984. case MP_BUS_MCA: /* MCA pin */
  985. {
  986. trigger = default_MCA_trigger(idx);
  987. break;
  988. }
  989. default:
  990. {
  991. printk(KERN_WARNING "broken BIOS!!\n");
  992. trigger = 1;
  993. break;
  994. }
  995. }
  996. #endif
  997. break;
  998. case 1: /* edge */
  999. {
  1000. trigger = 0;
  1001. break;
  1002. }
  1003. case 2: /* reserved */
  1004. {
  1005. printk(KERN_WARNING "broken BIOS!!\n");
  1006. trigger = 1;
  1007. break;
  1008. }
  1009. case 3: /* level */
  1010. {
  1011. trigger = 1;
  1012. break;
  1013. }
  1014. default: /* invalid */
  1015. {
  1016. printk(KERN_WARNING "broken BIOS!!\n");
  1017. trigger = 0;
  1018. break;
  1019. }
  1020. }
  1021. return trigger;
  1022. }
  1023. static inline int irq_polarity(int idx)
  1024. {
  1025. return MPBIOS_polarity(idx);
  1026. }
  1027. static inline int irq_trigger(int idx)
  1028. {
  1029. return MPBIOS_trigger(idx);
  1030. }
  1031. int (*ioapic_renumber_irq)(int ioapic, int irq);
  1032. static int pin_2_irq(int idx, int apic, int pin)
  1033. {
  1034. int irq, i;
  1035. int bus = mp_irqs[idx].mp_srcbus;
  1036. /*
  1037. * Debugging check, we are in big trouble if this message pops up!
  1038. */
  1039. if (mp_irqs[idx].mp_dstirq != pin)
  1040. printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
  1041. if (test_bit(bus, mp_bus_not_pci)) {
  1042. irq = mp_irqs[idx].mp_srcbusirq;
  1043. } else {
  1044. /*
  1045. * PCI IRQs are mapped in order
  1046. */
  1047. i = irq = 0;
  1048. while (i < apic)
  1049. irq += nr_ioapic_registers[i++];
  1050. irq += pin;
  1051. /*
  1052. * For MPS mode, so far only needed by ES7000 platform
  1053. */
  1054. if (ioapic_renumber_irq)
  1055. irq = ioapic_renumber_irq(apic, irq);
  1056. }
  1057. #ifdef CONFIG_X86_32
  1058. /*
  1059. * PCI IRQ command line redirection. Yes, limits are hardcoded.
  1060. */
  1061. if ((pin >= 16) && (pin <= 23)) {
  1062. if (pirq_entries[pin-16] != -1) {
  1063. if (!pirq_entries[pin-16]) {
  1064. apic_printk(APIC_VERBOSE, KERN_DEBUG
  1065. "disabling PIRQ%d\n", pin-16);
  1066. } else {
  1067. irq = pirq_entries[pin-16];
  1068. apic_printk(APIC_VERBOSE, KERN_DEBUG
  1069. "using PIRQ%d -> IRQ %d\n",
  1070. pin-16, irq);
  1071. }
  1072. }
  1073. }
  1074. #endif
  1075. return irq;
  1076. }
  1077. void lock_vector_lock(void)
  1078. {
  1079. /* Used to the online set of cpus does not change
  1080. * during assign_irq_vector.
  1081. */
  1082. spin_lock(&vector_lock);
  1083. }
  1084. void unlock_vector_lock(void)
  1085. {
  1086. spin_unlock(&vector_lock);
  1087. }
  1088. static int
  1089. __assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
  1090. {
  1091. /*
  1092. * NOTE! The local APIC isn't very good at handling
  1093. * multiple interrupts at the same interrupt level.
  1094. * As the interrupt level is determined by taking the
  1095. * vector number and shifting that right by 4, we
  1096. * want to spread these out a bit so that they don't
  1097. * all fall in the same interrupt level.
  1098. *
  1099. * Also, we've got to be careful not to trash gate
  1100. * 0x80, because int 0x80 is hm, kind of importantish. ;)
  1101. */
  1102. static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
  1103. unsigned int old_vector;
  1104. int cpu, err;
  1105. cpumask_var_t tmp_mask;
  1106. if ((cfg->move_in_progress) || cfg->move_cleanup_count)
  1107. return -EBUSY;
  1108. if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
  1109. return -ENOMEM;
  1110. old_vector = cfg->vector;
  1111. if (old_vector) {
  1112. cpumask_and(tmp_mask, mask, cpu_online_mask);
  1113. cpumask_and(tmp_mask, cfg->domain, tmp_mask);
  1114. if (!cpumask_empty(tmp_mask)) {
  1115. free_cpumask_var(tmp_mask);
  1116. return 0;
  1117. }
  1118. }
  1119. /* Only try and allocate irqs on cpus that are present */
  1120. err = -ENOSPC;
  1121. for_each_cpu_and(cpu, mask, cpu_online_mask) {
  1122. int new_cpu;
  1123. int vector, offset;
  1124. vector_allocation_domain(cpu, tmp_mask);
  1125. vector = current_vector;
  1126. offset = current_offset;
  1127. next:
  1128. vector += 8;
  1129. if (vector >= first_system_vector) {
  1130. /* If out of vectors on large boxen, must share them. */
  1131. offset = (offset + 1) % 8;
  1132. vector = FIRST_DEVICE_VECTOR + offset;
  1133. }
  1134. if (unlikely(current_vector == vector))
  1135. continue;
  1136. #ifdef CONFIG_X86_64
  1137. if (vector == IA32_SYSCALL_VECTOR)
  1138. goto next;
  1139. #else
  1140. if (vector == SYSCALL_VECTOR)
  1141. goto next;
  1142. #endif
  1143. for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
  1144. if (per_cpu(vector_irq, new_cpu)[vector] != -1)
  1145. goto next;
  1146. /* Found one! */
  1147. current_vector = vector;
  1148. current_offset = offset;
  1149. if (old_vector) {
  1150. cfg->move_in_progress = 1;
  1151. cpumask_copy(cfg->old_domain, cfg->domain);
  1152. }
  1153. for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
  1154. per_cpu(vector_irq, new_cpu)[vector] = irq;
  1155. cfg->vector = vector;
  1156. cpumask_copy(cfg->domain, tmp_mask);
  1157. err = 0;
  1158. break;
  1159. }
  1160. free_cpumask_var(tmp_mask);
  1161. return err;
  1162. }
  1163. static int
  1164. assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
  1165. {
  1166. int err;
  1167. unsigned long flags;
  1168. spin_lock_irqsave(&vector_lock, flags);
  1169. err = __assign_irq_vector(irq, cfg, mask);
  1170. spin_unlock_irqrestore(&vector_lock, flags);
  1171. return err;
  1172. }
  1173. static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
  1174. {
  1175. int cpu, vector;
  1176. BUG_ON(!cfg->vector);
  1177. vector = cfg->vector;
  1178. for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
  1179. per_cpu(vector_irq, cpu)[vector] = -1;
  1180. cfg->vector = 0;
  1181. cpumask_clear(cfg->domain);
  1182. if (likely(!cfg->move_in_progress))
  1183. return;
  1184. for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
  1185. for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
  1186. vector++) {
  1187. if (per_cpu(vector_irq, cpu)[vector] != irq)
  1188. continue;
  1189. per_cpu(vector_irq, cpu)[vector] = -1;
  1190. break;
  1191. }
  1192. }
  1193. cfg->move_in_progress = 0;
  1194. }
  1195. void __setup_vector_irq(int cpu)
  1196. {
  1197. /* Initialize vector_irq on a new cpu */
  1198. /* This function must be called with vector_lock held */
  1199. int irq, vector;
  1200. struct irq_cfg *cfg;
  1201. struct irq_desc *desc;
  1202. /* Mark the inuse vectors */
  1203. for_each_irq_desc(irq, desc) {
  1204. if (!desc)
  1205. continue;
  1206. cfg = desc->chip_data;
  1207. if (!cpumask_test_cpu(cpu, cfg->domain))
  1208. continue;
  1209. vector = cfg->vector;
  1210. per_cpu(vector_irq, cpu)[vector] = irq;
  1211. }
  1212. /* Mark the free vectors */
  1213. for (vector = 0; vector < NR_VECTORS; ++vector) {
  1214. irq = per_cpu(vector_irq, cpu)[vector];
  1215. if (irq < 0)
  1216. continue;
  1217. cfg = irq_cfg(irq);
  1218. if (!cpumask_test_cpu(cpu, cfg->domain))
  1219. per_cpu(vector_irq, cpu)[vector] = -1;
  1220. }
  1221. }
  1222. static struct irq_chip ioapic_chip;
  1223. #ifdef CONFIG_INTR_REMAP
  1224. static struct irq_chip ir_ioapic_chip;
  1225. #endif
  1226. #define IOAPIC_AUTO -1
  1227. #define IOAPIC_EDGE 0
  1228. #define IOAPIC_LEVEL 1
  1229. #ifdef CONFIG_X86_32
  1230. static inline int IO_APIC_irq_trigger(int irq)
  1231. {
  1232. int apic, idx, pin;
  1233. for (apic = 0; apic < nr_ioapics; apic++) {
  1234. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
  1235. idx = find_irq_entry(apic, pin, mp_INT);
  1236. if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
  1237. return irq_trigger(idx);
  1238. }
  1239. }
  1240. /*
  1241. * nonexistent IRQs are edge default
  1242. */
  1243. return 0;
  1244. }
  1245. #else
  1246. static inline int IO_APIC_irq_trigger(int irq)
  1247. {
  1248. return 1;
  1249. }
  1250. #endif
  1251. static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
  1252. {
  1253. if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
  1254. trigger == IOAPIC_LEVEL)
  1255. desc->status |= IRQ_LEVEL;
  1256. else
  1257. desc->status &= ~IRQ_LEVEL;
  1258. #ifdef CONFIG_INTR_REMAP
  1259. if (irq_remapped(irq)) {
  1260. desc->status |= IRQ_MOVE_PCNTXT;
  1261. if (trigger)
  1262. set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
  1263. handle_fasteoi_irq,
  1264. "fasteoi");
  1265. else
  1266. set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
  1267. handle_edge_irq, "edge");
  1268. return;
  1269. }
  1270. #endif
  1271. if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
  1272. trigger == IOAPIC_LEVEL)
  1273. set_irq_chip_and_handler_name(irq, &ioapic_chip,
  1274. handle_fasteoi_irq,
  1275. "fasteoi");
  1276. else
  1277. set_irq_chip_and_handler_name(irq, &ioapic_chip,
  1278. handle_edge_irq, "edge");
  1279. }
  1280. static int setup_ioapic_entry(int apic, int irq,
  1281. struct IO_APIC_route_entry *entry,
  1282. unsigned int destination, int trigger,
  1283. int polarity, int vector)
  1284. {
  1285. /*
  1286. * add it to the IO-APIC irq-routing table:
  1287. */
  1288. memset(entry,0,sizeof(*entry));
  1289. #ifdef CONFIG_INTR_REMAP
  1290. if (intr_remapping_enabled) {
  1291. struct intel_iommu *iommu = map_ioapic_to_ir(apic);
  1292. struct irte irte;
  1293. struct IR_IO_APIC_route_entry *ir_entry =
  1294. (struct IR_IO_APIC_route_entry *) entry;
  1295. int index;
  1296. if (!iommu)
  1297. panic("No mapping iommu for ioapic %d\n", apic);
  1298. index = alloc_irte(iommu, irq, 1);
  1299. if (index < 0)
  1300. panic("Failed to allocate IRTE for ioapic %d\n", apic);
  1301. memset(&irte, 0, sizeof(irte));
  1302. irte.present = 1;
  1303. irte.dst_mode = INT_DEST_MODE;
  1304. irte.trigger_mode = trigger;
  1305. irte.dlvry_mode = INT_DELIVERY_MODE;
  1306. irte.vector = vector;
  1307. irte.dest_id = IRTE_DEST(destination);
  1308. modify_irte(irq, &irte);
  1309. ir_entry->index2 = (index >> 15) & 0x1;
  1310. ir_entry->zero = 0;
  1311. ir_entry->format = 1;
  1312. ir_entry->index = (index & 0x7fff);
  1313. } else
  1314. #endif
  1315. {
  1316. entry->delivery_mode = INT_DELIVERY_MODE;
  1317. entry->dest_mode = INT_DEST_MODE;
  1318. entry->dest = destination;
  1319. }
  1320. entry->mask = 0; /* enable IRQ */
  1321. entry->trigger = trigger;
  1322. entry->polarity = polarity;
  1323. entry->vector = vector;
  1324. /* Mask level triggered irqs.
  1325. * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
  1326. */
  1327. if (trigger)
  1328. entry->mask = 1;
  1329. return 0;
  1330. }
  1331. static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_desc *desc,
  1332. int trigger, int polarity)
  1333. {
  1334. struct irq_cfg *cfg;
  1335. struct IO_APIC_route_entry entry;
  1336. unsigned int dest;
  1337. if (!IO_APIC_IRQ(irq))
  1338. return;
  1339. cfg = desc->chip_data;
  1340. if (assign_irq_vector(irq, cfg, TARGET_CPUS))
  1341. return;
  1342. dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
  1343. apic_printk(APIC_VERBOSE,KERN_DEBUG
  1344. "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
  1345. "IRQ %d Mode:%i Active:%i)\n",
  1346. apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
  1347. irq, trigger, polarity);
  1348. if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
  1349. dest, trigger, polarity, cfg->vector)) {
  1350. printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
  1351. mp_ioapics[apic].mp_apicid, pin);
  1352. __clear_irq_vector(irq, cfg);
  1353. return;
  1354. }
  1355. ioapic_register_intr(irq, desc, trigger);
  1356. if (irq < NR_IRQS_LEGACY)
  1357. disable_8259A_irq(irq);
  1358. ioapic_write_entry(apic, pin, entry);
  1359. }
  1360. static void __init setup_IO_APIC_irqs(void)
  1361. {
  1362. int apic, pin, idx, irq;
  1363. int notcon = 0;
  1364. struct irq_desc *desc;
  1365. struct irq_cfg *cfg;
  1366. int cpu = boot_cpu_id;
  1367. apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
  1368. for (apic = 0; apic < nr_ioapics; apic++) {
  1369. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
  1370. idx = find_irq_entry(apic, pin, mp_INT);
  1371. if (idx == -1) {
  1372. if (!notcon) {
  1373. notcon = 1;
  1374. apic_printk(APIC_VERBOSE,
  1375. KERN_DEBUG " %d-%d",
  1376. mp_ioapics[apic].mp_apicid,
  1377. pin);
  1378. } else
  1379. apic_printk(APIC_VERBOSE, " %d-%d",
  1380. mp_ioapics[apic].mp_apicid,
  1381. pin);
  1382. continue;
  1383. }
  1384. if (notcon) {
  1385. apic_printk(APIC_VERBOSE,
  1386. " (apicid-pin) not connected\n");
  1387. notcon = 0;
  1388. }
  1389. irq = pin_2_irq(idx, apic, pin);
  1390. #ifdef CONFIG_X86_32
  1391. if (multi_timer_check(apic, irq))
  1392. continue;
  1393. #endif
  1394. desc = irq_to_desc_alloc_cpu(irq, cpu);
  1395. if (!desc) {
  1396. printk(KERN_INFO "can not get irq_desc for %d\n", irq);
  1397. continue;
  1398. }
  1399. cfg = desc->chip_data;
  1400. add_pin_to_irq_cpu(cfg, cpu, apic, pin);
  1401. setup_IO_APIC_irq(apic, pin, irq, desc,
  1402. irq_trigger(idx), irq_polarity(idx));
  1403. }
  1404. }
  1405. if (notcon)
  1406. apic_printk(APIC_VERBOSE,
  1407. " (apicid-pin) not connected\n");
  1408. }
  1409. /*
  1410. * Set up the timer pin, possibly with the 8259A-master behind.
  1411. */
  1412. static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
  1413. int vector)
  1414. {
  1415. struct IO_APIC_route_entry entry;
  1416. #ifdef CONFIG_INTR_REMAP
  1417. if (intr_remapping_enabled)
  1418. return;
  1419. #endif
  1420. memset(&entry, 0, sizeof(entry));
  1421. /*
  1422. * We use logical delivery to get the timer IRQ
  1423. * to the first CPU.
  1424. */
  1425. entry.dest_mode = INT_DEST_MODE;
  1426. entry.mask = 1; /* mask IRQ now */
  1427. entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
  1428. entry.delivery_mode = INT_DELIVERY_MODE;
  1429. entry.polarity = 0;
  1430. entry.trigger = 0;
  1431. entry.vector = vector;
  1432. /*
  1433. * The timer IRQ doesn't have to know that behind the
  1434. * scene we may have a 8259A-master in AEOI mode ...
  1435. */
  1436. set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
  1437. /*
  1438. * Add it to the IO-APIC irq-routing table:
  1439. */
  1440. ioapic_write_entry(apic, pin, entry);
  1441. }
  1442. __apicdebuginit(void) print_IO_APIC(void)
  1443. {
  1444. int apic, i;
  1445. union IO_APIC_reg_00 reg_00;
  1446. union IO_APIC_reg_01 reg_01;
  1447. union IO_APIC_reg_02 reg_02;
  1448. union IO_APIC_reg_03 reg_03;
  1449. unsigned long flags;
  1450. struct irq_cfg *cfg;
  1451. struct irq_desc *desc;
  1452. unsigned int irq;
  1453. if (apic_verbosity == APIC_QUIET)
  1454. return;
  1455. printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
  1456. for (i = 0; i < nr_ioapics; i++)
  1457. printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
  1458. mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
  1459. /*
  1460. * We are a bit conservative about what we expect. We have to
  1461. * know about every hardware change ASAP.
  1462. */
  1463. printk(KERN_INFO "testing the IO APIC.......................\n");
  1464. for (apic = 0; apic < nr_ioapics; apic++) {
  1465. spin_lock_irqsave(&ioapic_lock, flags);
  1466. reg_00.raw = io_apic_read(apic, 0);
  1467. reg_01.raw = io_apic_read(apic, 1);
  1468. if (reg_01.bits.version >= 0x10)
  1469. reg_02.raw = io_apic_read(apic, 2);
  1470. if (reg_01.bits.version >= 0x20)
  1471. reg_03.raw = io_apic_read(apic, 3);
  1472. spin_unlock_irqrestore(&ioapic_lock, flags);
  1473. printk("\n");
  1474. printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
  1475. printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
  1476. printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
  1477. printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
  1478. printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
  1479. printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
  1480. printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries);
  1481. printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
  1482. printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version);
  1483. /*
  1484. * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
  1485. * but the value of reg_02 is read as the previous read register
  1486. * value, so ignore it if reg_02 == reg_01.
  1487. */
  1488. if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
  1489. printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
  1490. printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
  1491. }
  1492. /*
  1493. * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
  1494. * or reg_03, but the value of reg_0[23] is read as the previous read
  1495. * register value, so ignore it if reg_03 == reg_0[12].
  1496. */
  1497. if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
  1498. reg_03.raw != reg_01.raw) {
  1499. printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
  1500. printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
  1501. }
  1502. printk(KERN_DEBUG ".... IRQ redirection table:\n");
  1503. printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
  1504. " Stat Dmod Deli Vect: \n");
  1505. for (i = 0; i <= reg_01.bits.entries; i++) {
  1506. struct IO_APIC_route_entry entry;
  1507. entry = ioapic_read_entry(apic, i);
  1508. printk(KERN_DEBUG " %02x %03X ",
  1509. i,
  1510. entry.dest
  1511. );
  1512. printk("%1d %1d %1d %1d %1d %1d %1d %02X\n",
  1513. entry.mask,
  1514. entry.trigger,
  1515. entry.irr,
  1516. entry.polarity,
  1517. entry.delivery_status,
  1518. entry.dest_mode,
  1519. entry.delivery_mode,
  1520. entry.vector
  1521. );
  1522. }
  1523. }
  1524. printk(KERN_DEBUG "IRQ to pin mappings:\n");
  1525. for_each_irq_desc(irq, desc) {
  1526. struct irq_pin_list *entry;
  1527. if (!desc)
  1528. continue;
  1529. cfg = desc->chip_data;
  1530. entry = cfg->irq_2_pin;
  1531. if (!entry)
  1532. continue;
  1533. printk(KERN_DEBUG "IRQ%d ", irq);
  1534. for (;;) {
  1535. printk("-> %d:%d", entry->apic, entry->pin);
  1536. if (!entry->next)
  1537. break;
  1538. entry = entry->next;
  1539. }
  1540. printk("\n");
  1541. }
  1542. printk(KERN_INFO ".................................... done.\n");
  1543. return;
  1544. }
  1545. __apicdebuginit(void) print_APIC_bitfield(int base)
  1546. {
  1547. unsigned int v;
  1548. int i, j;
  1549. if (apic_verbosity == APIC_QUIET)
  1550. return;
  1551. printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
  1552. for (i = 0; i < 8; i++) {
  1553. v = apic_read(base + i*0x10);
  1554. for (j = 0; j < 32; j++) {
  1555. if (v & (1<<j))
  1556. printk("1");
  1557. else
  1558. printk("0");
  1559. }
  1560. printk("\n");
  1561. }
  1562. }
  1563. __apicdebuginit(void) print_local_APIC(void *dummy)
  1564. {
  1565. unsigned int v, ver, maxlvt;
  1566. u64 icr;
  1567. if (apic_verbosity == APIC_QUIET)
  1568. return;
  1569. printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
  1570. smp_processor_id(), hard_smp_processor_id());
  1571. v = apic_read(APIC_ID);
  1572. printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
  1573. v = apic_read(APIC_LVR);
  1574. printk(KERN_INFO "... APIC VERSION: %08x\n", v);
  1575. ver = GET_APIC_VERSION(v);
  1576. maxlvt = lapic_get_maxlvt();
  1577. v = apic_read(APIC_TASKPRI);
  1578. printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
  1579. if (APIC_INTEGRATED(ver)) { /* !82489DX */
  1580. if (!APIC_XAPIC(ver)) {
  1581. v = apic_read(APIC_ARBPRI);
  1582. printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
  1583. v & APIC_ARBPRI_MASK);
  1584. }
  1585. v = apic_read(APIC_PROCPRI);
  1586. printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
  1587. }
  1588. /*
  1589. * Remote read supported only in the 82489DX and local APIC for
  1590. * Pentium processors.
  1591. */
  1592. if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
  1593. v = apic_read(APIC_RRR);
  1594. printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
  1595. }
  1596. v = apic_read(APIC_LDR);
  1597. printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
  1598. if (!x2apic_enabled()) {
  1599. v = apic_read(APIC_DFR);
  1600. printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
  1601. }
  1602. v = apic_read(APIC_SPIV);
  1603. printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
  1604. printk(KERN_DEBUG "... APIC ISR field:\n");
  1605. print_APIC_bitfield(APIC_ISR);
  1606. printk(KERN_DEBUG "... APIC TMR field:\n");
  1607. print_APIC_bitfield(APIC_TMR);
  1608. printk(KERN_DEBUG "... APIC IRR field:\n");
  1609. print_APIC_bitfield(APIC_IRR);
  1610. if (APIC_INTEGRATED(ver)) { /* !82489DX */
  1611. if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
  1612. apic_write(APIC_ESR, 0);
  1613. v = apic_read(APIC_ESR);
  1614. printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
  1615. }
  1616. icr = apic_icr_read();
  1617. printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
  1618. printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
  1619. v = apic_read(APIC_LVTT);
  1620. printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
  1621. if (maxlvt > 3) { /* PC is LVT#4. */
  1622. v = apic_read(APIC_LVTPC);
  1623. printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
  1624. }
  1625. v = apic_read(APIC_LVT0);
  1626. printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
  1627. v = apic_read(APIC_LVT1);
  1628. printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
  1629. if (maxlvt > 2) { /* ERR is LVT#3. */
  1630. v = apic_read(APIC_LVTERR);
  1631. printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
  1632. }
  1633. v = apic_read(APIC_TMICT);
  1634. printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
  1635. v = apic_read(APIC_TMCCT);
  1636. printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
  1637. v = apic_read(APIC_TDCR);
  1638. printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
  1639. printk("\n");
  1640. }
  1641. __apicdebuginit(void) print_all_local_APICs(void)
  1642. {
  1643. int cpu;
  1644. preempt_disable();
  1645. for_each_online_cpu(cpu)
  1646. smp_call_function_single(cpu, print_local_APIC, NULL, 1);
  1647. preempt_enable();
  1648. }
  1649. __apicdebuginit(void) print_PIC(void)
  1650. {
  1651. unsigned int v;
  1652. unsigned long flags;
  1653. if (apic_verbosity == APIC_QUIET)
  1654. return;
  1655. printk(KERN_DEBUG "\nprinting PIC contents\n");
  1656. spin_lock_irqsave(&i8259A_lock, flags);
  1657. v = inb(0xa1) << 8 | inb(0x21);
  1658. printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
  1659. v = inb(0xa0) << 8 | inb(0x20);
  1660. printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
  1661. outb(0x0b,0xa0);
  1662. outb(0x0b,0x20);
  1663. v = inb(0xa0) << 8 | inb(0x20);
  1664. outb(0x0a,0xa0);
  1665. outb(0x0a,0x20);
  1666. spin_unlock_irqrestore(&i8259A_lock, flags);
  1667. printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
  1668. v = inb(0x4d1) << 8 | inb(0x4d0);
  1669. printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
  1670. }
  1671. __apicdebuginit(int) print_all_ICs(void)
  1672. {
  1673. print_PIC();
  1674. print_all_local_APICs();
  1675. print_IO_APIC();
  1676. return 0;
  1677. }
  1678. fs_initcall(print_all_ICs);
  1679. /* Where if anywhere is the i8259 connect in external int mode */
  1680. static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
  1681. void __init enable_IO_APIC(void)
  1682. {
  1683. union IO_APIC_reg_01 reg_01;
  1684. int i8259_apic, i8259_pin;
  1685. int apic;
  1686. unsigned long flags;
  1687. #ifdef CONFIG_X86_32
  1688. int i;
  1689. if (!pirqs_enabled)
  1690. for (i = 0; i < MAX_PIRQS; i++)
  1691. pirq_entries[i] = -1;
  1692. #endif
  1693. /*
  1694. * The number of IO-APIC IRQ registers (== #pins):
  1695. */
  1696. for (apic = 0; apic < nr_ioapics; apic++) {
  1697. spin_lock_irqsave(&ioapic_lock, flags);
  1698. reg_01.raw = io_apic_read(apic, 1);
  1699. spin_unlock_irqrestore(&ioapic_lock, flags);
  1700. nr_ioapic_registers[apic] = reg_01.bits.entries+1;
  1701. }
  1702. for(apic = 0; apic < nr_ioapics; apic++) {
  1703. int pin;
  1704. /* See if any of the pins is in ExtINT mode */
  1705. for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
  1706. struct IO_APIC_route_entry entry;
  1707. entry = ioapic_read_entry(apic, pin);
  1708. /* If the interrupt line is enabled and in ExtInt mode
  1709. * I have found the pin where the i8259 is connected.
  1710. */
  1711. if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
  1712. ioapic_i8259.apic = apic;
  1713. ioapic_i8259.pin = pin;
  1714. goto found_i8259;
  1715. }
  1716. }
  1717. }
  1718. found_i8259:
  1719. /* Look to see what if the MP table has reported the ExtINT */
  1720. /* If we could not find the appropriate pin by looking at the ioapic
  1721. * the i8259 probably is not connected the ioapic but give the
  1722. * mptable a chance anyway.
  1723. */
  1724. i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
  1725. i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
  1726. /* Trust the MP table if nothing is setup in the hardware */
  1727. if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
  1728. printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
  1729. ioapic_i8259.pin = i8259_pin;
  1730. ioapic_i8259.apic = i8259_apic;
  1731. }
  1732. /* Complain if the MP table and the hardware disagree */
  1733. if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
  1734. (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
  1735. {
  1736. printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
  1737. }
  1738. /*
  1739. * Do not trust the IO-APIC being empty at bootup
  1740. */
  1741. clear_IO_APIC();
  1742. }
  1743. /*
  1744. * Not an __init, needed by the reboot code
  1745. */
  1746. void disable_IO_APIC(void)
  1747. {
  1748. /*
  1749. * Clear the IO-APIC before rebooting:
  1750. */
  1751. clear_IO_APIC();
  1752. /*
  1753. * If the i8259 is routed through an IOAPIC
  1754. * Put that IOAPIC in virtual wire mode
  1755. * so legacy interrupts can be delivered.
  1756. */
  1757. if (ioapic_i8259.pin != -1) {
  1758. struct IO_APIC_route_entry entry;
  1759. memset(&entry, 0, sizeof(entry));
  1760. entry.mask = 0; /* Enabled */
  1761. entry.trigger = 0; /* Edge */
  1762. entry.irr = 0;
  1763. entry.polarity = 0; /* High */
  1764. entry.delivery_status = 0;
  1765. entry.dest_mode = 0; /* Physical */
  1766. entry.delivery_mode = dest_ExtINT; /* ExtInt */
  1767. entry.vector = 0;
  1768. entry.dest = read_apic_id();
  1769. /*
  1770. * Add it to the IO-APIC irq-routing table:
  1771. */
  1772. ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
  1773. }
  1774. disconnect_bsp_APIC(ioapic_i8259.pin != -1);
  1775. }
  1776. #ifdef CONFIG_X86_32
  1777. /*
  1778. * function to set the IO-APIC physical IDs based on the
  1779. * values stored in the MPC table.
  1780. *
  1781. * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
  1782. */
  1783. static void __init setup_ioapic_ids_from_mpc(void)
  1784. {
  1785. union IO_APIC_reg_00 reg_00;
  1786. physid_mask_t phys_id_present_map;
  1787. int apic;
  1788. int i;
  1789. unsigned char old_id;
  1790. unsigned long flags;
  1791. if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
  1792. return;
  1793. /*
  1794. * Don't check I/O APIC IDs for xAPIC systems. They have
  1795. * no meaning without the serial APIC bus.
  1796. */
  1797. if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
  1798. || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
  1799. return;
  1800. /*
  1801. * This is broken; anything with a real cpu count has to
  1802. * circumvent this idiocy regardless.
  1803. */
  1804. phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);
  1805. /*
  1806. * Set the IOAPIC ID to the value stored in the MPC table.
  1807. */
  1808. for (apic = 0; apic < nr_ioapics; apic++) {
  1809. /* Read the register 0 value */
  1810. spin_lock_irqsave(&ioapic_lock, flags);
  1811. reg_00.raw = io_apic_read(apic, 0);
  1812. spin_unlock_irqrestore(&ioapic_lock, flags);
  1813. old_id = mp_ioapics[apic].mp_apicid;
  1814. if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
  1815. printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
  1816. apic, mp_ioapics[apic].mp_apicid);
  1817. printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
  1818. reg_00.bits.ID);
  1819. mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
  1820. }
  1821. /*
  1822. * Sanity check, is the ID really free? Every APIC in a
  1823. * system must have a unique ID or we get lots of nice
  1824. * 'stuck on smp_invalidate_needed IPI wait' messages.
  1825. */
  1826. if (check_apicid_used(phys_id_present_map,
  1827. mp_ioapics[apic].mp_apicid)) {
  1828. printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
  1829. apic, mp_ioapics[apic].mp_apicid);
  1830. for (i = 0; i < get_physical_broadcast(); i++)
  1831. if (!physid_isset(i, phys_id_present_map))
  1832. break;
  1833. if (i >= get_physical_broadcast())
  1834. panic("Max APIC ID exceeded!\n");
  1835. printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
  1836. i);
  1837. physid_set(i, phys_id_present_map);
  1838. mp_ioapics[apic].mp_apicid = i;
  1839. } else {
  1840. physid_mask_t tmp;
  1841. tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
  1842. apic_printk(APIC_VERBOSE, "Setting %d in the "
  1843. "phys_id_present_map\n",
  1844. mp_ioapics[apic].mp_apicid);
  1845. physids_or(phys_id_present_map, phys_id_present_map, tmp);
  1846. }
  1847. /*
  1848. * We need to adjust the IRQ routing table
  1849. * if the ID changed.
  1850. */
  1851. if (old_id != mp_ioapics[apic].mp_apicid)
  1852. for (i = 0; i < mp_irq_entries; i++)
  1853. if (mp_irqs[i].mp_dstapic == old_id)
  1854. mp_irqs[i].mp_dstapic
  1855. = mp_ioapics[apic].mp_apicid;
  1856. /*
  1857. * Read the right value from the MPC table and
  1858. * write it into the ID register.
  1859. */
  1860. apic_printk(APIC_VERBOSE, KERN_INFO
  1861. "...changing IO-APIC physical APIC ID to %d ...",
  1862. mp_ioapics[apic].mp_apicid);
  1863. reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
  1864. spin_lock_irqsave(&ioapic_lock, flags);
  1865. io_apic_write(apic, 0, reg_00.raw);
  1866. spin_unlock_irqrestore(&ioapic_lock, flags);
  1867. /*
  1868. * Sanity check
  1869. */
  1870. spin_lock_irqsave(&ioapic_lock, flags);
  1871. reg_00.raw = io_apic_read(apic, 0);
  1872. spin_unlock_irqrestore(&ioapic_lock, flags);
  1873. if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
  1874. printk("could not set ID!\n");
  1875. else
  1876. apic_printk(APIC_VERBOSE, " ok.\n");
  1877. }
  1878. }
  1879. #endif
  1880. int no_timer_check __initdata;
  1881. static int __init notimercheck(char *s)
  1882. {
  1883. no_timer_check = 1;
  1884. return 1;
  1885. }
  1886. __setup("no_timer_check", notimercheck);
  1887. /*
  1888. * There is a nasty bug in some older SMP boards, their mptable lies
  1889. * about the timer IRQ. We do the following to work around the situation:
  1890. *
  1891. * - timer IRQ defaults to IO-APIC IRQ
  1892. * - if this function detects that timer IRQs are defunct, then we fall
  1893. * back to ISA timer IRQs
  1894. */
  1895. static int __init timer_irq_works(void)
  1896. {
  1897. unsigned long t1 = jiffies;
  1898. unsigned long flags;
  1899. if (no_timer_check)
  1900. return 1;
  1901. local_save_flags(flags);
  1902. local_irq_enable();
  1903. /* Let ten ticks pass... */
  1904. mdelay((10 * 1000) / HZ);
  1905. local_irq_restore(flags);
  1906. /*
  1907. * Expect a few ticks at least, to be sure some possible
  1908. * glue logic does not lock up after one or two first
  1909. * ticks in a non-ExtINT mode. Also the local APIC
  1910. * might have cached one ExtINT interrupt. Finally, at
  1911. * least one tick may be lost due to delays.
  1912. */
  1913. /* jiffies wrap? */
  1914. if (time_after(jiffies, t1 + 4))
  1915. return 1;
  1916. return 0;
  1917. }
  1918. /*
  1919. * In the SMP+IOAPIC case it might happen that there are an unspecified
  1920. * number of pending IRQ events unhandled. These cases are very rare,
  1921. * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
  1922. * better to do it this way as thus we do not have to be aware of
  1923. * 'pending' interrupts in the IRQ path, except at this point.
  1924. */
  1925. /*
  1926. * Edge triggered needs to resend any interrupt
  1927. * that was delayed but this is now handled in the device
  1928. * independent code.
  1929. */
  1930. /*
  1931. * Starting up a edge-triggered IO-APIC interrupt is
  1932. * nasty - we need to make sure that we get the edge.
  1933. * If it is already asserted for some reason, we need
  1934. * return 1 to indicate that is was pending.
  1935. *
  1936. * This is not complete - we should be able to fake
  1937. * an edge even if it isn't on the 8259A...
  1938. */
  1939. static unsigned int startup_ioapic_irq(unsigned int irq)
  1940. {
  1941. int was_pending = 0;
  1942. unsigned long flags;
  1943. struct irq_cfg *cfg;
  1944. spin_lock_irqsave(&ioapic_lock, flags);
  1945. if (irq < NR_IRQS_LEGACY) {
  1946. disable_8259A_irq(irq);
  1947. if (i8259A_irq_pending(irq))
  1948. was_pending = 1;
  1949. }
  1950. cfg = irq_cfg(irq);
  1951. __unmask_IO_APIC_irq(cfg);
  1952. spin_unlock_irqrestore(&ioapic_lock, flags);
  1953. return was_pending;
  1954. }
  1955. #ifdef CONFIG_X86_64
  1956. static int ioapic_retrigger_irq(unsigned int irq)
  1957. {
  1958. struct irq_cfg *cfg = irq_cfg(irq);
  1959. unsigned long flags;
  1960. spin_lock_irqsave(&vector_lock, flags);
  1961. send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
  1962. spin_unlock_irqrestore(&vector_lock, flags);
  1963. return 1;
  1964. }
  1965. #else
  1966. static int ioapic_retrigger_irq(unsigned int irq)
  1967. {
  1968. send_IPI_self(irq_cfg(irq)->vector);
  1969. return 1;
  1970. }
  1971. #endif
  1972. /*
  1973. * Level and edge triggered IO-APIC interrupts need different handling,
  1974. * so we use two separate IRQ descriptors. Edge triggered IRQs can be
  1975. * handled with the level-triggered descriptor, but that one has slightly
  1976. * more overhead. Level-triggered interrupts cannot be handled with the
  1977. * edge-triggered handler, without risking IRQ storms and other ugly
  1978. * races.
  1979. */
  1980. #ifdef CONFIG_SMP
  1981. #ifdef CONFIG_INTR_REMAP
  1982. static void ir_irq_migration(struct work_struct *work);
  1983. static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
  1984. /*
  1985. * Migrate the IO-APIC irq in the presence of intr-remapping.
  1986. *
  1987. * For edge triggered, irq migration is a simple atomic update(of vector
  1988. * and cpu destination) of IRTE and flush the hardware cache.
  1989. *
  1990. * For level triggered, we need to modify the io-apic RTE aswell with the update
  1991. * vector information, along with modifying IRTE with vector and destination.
  1992. * So irq migration for level triggered is little bit more complex compared to
  1993. * edge triggered migration. But the good news is, we use the same algorithm
  1994. * for level triggered migration as we have today, only difference being,
  1995. * we now initiate the irq migration from process context instead of the
  1996. * interrupt context.
  1997. *
  1998. * In future, when we do a directed EOI (combined with cpu EOI broadcast
  1999. * suppression) to the IO-APIC, level triggered irq migration will also be
  2000. * as simple as edge triggered migration and we can do the irq migration
  2001. * with a simple atomic update to IO-APIC RTE.
  2002. */
  2003. static void
  2004. migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
  2005. {
  2006. struct irq_cfg *cfg;
  2007. struct irte irte;
  2008. int modify_ioapic_rte;
  2009. unsigned int dest;
  2010. unsigned long flags;
  2011. unsigned int irq;
  2012. if (!cpumask_intersects(mask, cpu_online_mask))
  2013. return;
  2014. irq = desc->irq;
  2015. if (get_irte(irq, &irte))
  2016. return;
  2017. cfg = desc->chip_data;
  2018. if (assign_irq_vector(irq, cfg, mask))
  2019. return;
  2020. set_extra_move_desc(desc, mask);
  2021. dest = cpu_mask_to_apicid_and(cfg->domain, mask);
  2022. modify_ioapic_rte = desc->status & IRQ_LEVEL;
  2023. if (modify_ioapic_rte) {
  2024. spin_lock_irqsave(&ioapic_lock, flags);
  2025. __target_IO_APIC_irq(irq, dest, cfg);
  2026. spin_unlock_irqrestore(&ioapic_lock, flags);
  2027. }
  2028. irte.vector = cfg->vector;
  2029. irte.dest_id = IRTE_DEST(dest);
  2030. /*
  2031. * Modified the IRTE and flushes the Interrupt entry cache.
  2032. */
  2033. modify_irte(irq, &irte);
  2034. if (cfg->move_in_progress)
  2035. send_cleanup_vector(cfg);
  2036. cpumask_copy(&desc->affinity, mask);
  2037. }
  2038. static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
  2039. {
  2040. int ret = -1;
  2041. struct irq_cfg *cfg = desc->chip_data;
  2042. mask_IO_APIC_irq_desc(desc);
  2043. if (io_apic_level_ack_pending(cfg)) {
  2044. /*
  2045. * Interrupt in progress. Migrating irq now will change the
  2046. * vector information in the IO-APIC RTE and that will confuse
  2047. * the EOI broadcast performed by cpu.
  2048. * So, delay the irq migration to the next instance.
  2049. */
  2050. schedule_delayed_work(&ir_migration_work, 1);
  2051. goto unmask;
  2052. }
  2053. /* everthing is clear. we have right of way */
  2054. migrate_ioapic_irq_desc(desc, &desc->pending_mask);
  2055. ret = 0;
  2056. desc->status &= ~IRQ_MOVE_PENDING;
  2057. cpumask_clear(&desc->pending_mask);
  2058. unmask:
  2059. unmask_IO_APIC_irq_desc(desc);
  2060. return ret;
  2061. }
  2062. static void ir_irq_migration(struct work_struct *work)
  2063. {
  2064. unsigned int irq;
  2065. struct irq_desc *desc;
  2066. for_each_irq_desc(irq, desc) {
  2067. if (!desc)
  2068. continue;
  2069. if (desc->status & IRQ_MOVE_PENDING) {
  2070. unsigned long flags;
  2071. spin_lock_irqsave(&desc->lock, flags);
  2072. if (!desc->chip->set_affinity ||
  2073. !(desc->status & IRQ_MOVE_PENDING)) {
  2074. desc->status &= ~IRQ_MOVE_PENDING;
  2075. spin_unlock_irqrestore(&desc->lock, flags);
  2076. continue;
  2077. }
  2078. desc->chip->set_affinity(irq, &desc->pending_mask);
  2079. spin_unlock_irqrestore(&desc->lock, flags);
  2080. }
  2081. }
  2082. }
  2083. /*
  2084. * Migrates the IRQ destination in the process context.
  2085. */
  2086. static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
  2087. const struct cpumask *mask)
  2088. {
  2089. if (desc->status & IRQ_LEVEL) {
  2090. desc->status |= IRQ_MOVE_PENDING;
  2091. cpumask_copy(&desc->pending_mask, mask);
  2092. migrate_irq_remapped_level_desc(desc);
  2093. return;
  2094. }
  2095. migrate_ioapic_irq_desc(desc, mask);
  2096. }
  2097. static void set_ir_ioapic_affinity_irq(unsigned int irq,
  2098. const struct cpumask *mask)
  2099. {
  2100. struct irq_desc *desc = irq_to_desc(irq);
  2101. set_ir_ioapic_affinity_irq_desc(desc, mask);
  2102. }
  2103. #endif
  2104. asmlinkage void smp_irq_move_cleanup_interrupt(void)
  2105. {
  2106. unsigned vector, me;
  2107. ack_APIC_irq();
  2108. #ifdef CONFIG_X86_64
  2109. exit_idle();
  2110. #endif
  2111. irq_enter();
  2112. me = smp_processor_id();
  2113. for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
  2114. unsigned int irq;
  2115. struct irq_desc *desc;
  2116. struct irq_cfg *cfg;
  2117. irq = __get_cpu_var(vector_irq)[vector];
  2118. if (irq == -1)
  2119. continue;
  2120. desc = irq_to_desc(irq);
  2121. if (!desc)
  2122. continue;
  2123. cfg = irq_cfg(irq);
  2124. spin_lock(&desc->lock);
  2125. if (!cfg->move_cleanup_count)
  2126. goto unlock;
  2127. if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
  2128. goto unlock;
  2129. __get_cpu_var(vector_irq)[vector] = -1;
  2130. cfg->move_cleanup_count--;
  2131. unlock:
  2132. spin_unlock(&desc->lock);
  2133. }
  2134. irq_exit();
  2135. }
  2136. static void irq_complete_move(struct irq_desc **descp)
  2137. {
  2138. struct irq_desc *desc = *descp;
  2139. struct irq_cfg *cfg = desc->chip_data;
  2140. unsigned vector, me;
  2141. if (likely(!cfg->move_in_progress)) {
  2142. #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
  2143. if (likely(!cfg->move_desc_pending))
  2144. return;
  2145. /* domain is not change, but affinity is changed */
  2146. me = smp_processor_id();
  2147. if (cpu_isset(me, desc->affinity)) {
  2148. *descp = desc = move_irq_desc(desc, me);
  2149. /* get the new one */
  2150. cfg = desc->chip_data;
  2151. cfg->move_desc_pending = 0;
  2152. }
  2153. #endif
  2154. return;
  2155. }
  2156. vector = ~get_irq_regs()->orig_ax;
  2157. me = smp_processor_id();
  2158. #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
  2159. *descp = desc = move_irq_desc(desc, me);
  2160. /* get the new one */
  2161. cfg = desc->chip_data;
  2162. #endif
  2163. if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
  2164. send_cleanup_vector(cfg);
  2165. }
  2166. #else
  2167. static inline void irq_complete_move(struct irq_desc **descp) {}
  2168. #endif
  2169. #ifdef CONFIG_INTR_REMAP
  2170. static void ack_x2apic_level(unsigned int irq)
  2171. {
  2172. ack_x2APIC_irq();
  2173. }
  2174. static void ack_x2apic_edge(unsigned int irq)
  2175. {
  2176. ack_x2APIC_irq();
  2177. }
  2178. #endif
  2179. static void ack_apic_edge(unsigned int irq)
  2180. {
  2181. struct irq_desc *desc = irq_to_desc(irq);
  2182. irq_complete_move(&desc);
  2183. move_native_irq(irq);
  2184. ack_APIC_irq();
  2185. }
  2186. atomic_t irq_mis_count;
  2187. static void ack_apic_level(unsigned int irq)
  2188. {
  2189. struct irq_desc *desc = irq_to_desc(irq);
  2190. #ifdef CONFIG_X86_32
  2191. unsigned long v;
  2192. int i;
  2193. #endif
  2194. struct irq_cfg *cfg;
  2195. int do_unmask_irq = 0;
  2196. irq_complete_move(&desc);
  2197. #ifdef CONFIG_GENERIC_PENDING_IRQ
  2198. /* If we are moving the irq we need to mask it */
  2199. if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
  2200. do_unmask_irq = 1;
  2201. mask_IO_APIC_irq_desc(desc);
  2202. }
  2203. #endif
  2204. #ifdef CONFIG_X86_32
  2205. /*
  2206. * It appears there is an erratum which affects at least version 0x11
  2207. * of I/O APIC (that's the 82093AA and cores integrated into various
  2208. * chipsets). Under certain conditions a level-triggered interrupt is
  2209. * erroneously delivered as edge-triggered one but the respective IRR
  2210. * bit gets set nevertheless. As a result the I/O unit expects an EOI
  2211. * message but it will never arrive and further interrupts are blocked
  2212. * from the source. The exact reason is so far unknown, but the
  2213. * phenomenon was observed when two consecutive interrupt requests
  2214. * from a given source get delivered to the same CPU and the source is
  2215. * temporarily disabled in between.
  2216. *
  2217. * A workaround is to simulate an EOI message manually. We achieve it
  2218. * by setting the trigger mode to edge and then to level when the edge
  2219. * trigger mode gets detected in the TMR of a local APIC for a
  2220. * level-triggered interrupt. We mask the source for the time of the
  2221. * operation to prevent an edge-triggered interrupt escaping meanwhile.
  2222. * The idea is from Manfred Spraul. --macro
  2223. */
  2224. cfg = desc->chip_data;
  2225. i = cfg->vector;
  2226. v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
  2227. #endif
  2228. /*
  2229. * We must acknowledge the irq before we move it or the acknowledge will
  2230. * not propagate properly.
  2231. */
  2232. ack_APIC_irq();
  2233. /* Now we can move and renable the irq */
  2234. if (unlikely(do_unmask_irq)) {
  2235. /* Only migrate the irq if the ack has been received.
  2236. *
  2237. * On rare occasions the broadcast level triggered ack gets
  2238. * delayed going to ioapics, and if we reprogram the
  2239. * vector while Remote IRR is still set the irq will never
  2240. * fire again.
  2241. *
  2242. * To prevent this scenario we read the Remote IRR bit
  2243. * of the ioapic. This has two effects.
  2244. * - On any sane system the read of the ioapic will
  2245. * flush writes (and acks) going to the ioapic from
  2246. * this cpu.
  2247. * - We get to see if the ACK has actually been delivered.
  2248. *
  2249. * Based on failed experiments of reprogramming the
  2250. * ioapic entry from outside of irq context starting
  2251. * with masking the ioapic entry and then polling until
  2252. * Remote IRR was clear before reprogramming the
  2253. * ioapic I don't trust the Remote IRR bit to be
  2254. * completey accurate.
  2255. *
  2256. * However there appears to be no other way to plug
  2257. * this race, so if the Remote IRR bit is not
  2258. * accurate and is causing problems then it is a hardware bug
  2259. * and you can go talk to the chipset vendor about it.
  2260. */
  2261. cfg = desc->chip_data;
  2262. if (!io_apic_level_ack_pending(cfg))
  2263. move_masked_irq(irq);
  2264. unmask_IO_APIC_irq_desc(desc);
  2265. }
  2266. #ifdef CONFIG_X86_32
  2267. if (!(v & (1 << (i & 0x1f)))) {
  2268. atomic_inc(&irq_mis_count);
  2269. spin_lock(&ioapic_lock);
  2270. __mask_and_edge_IO_APIC_irq(cfg);
  2271. __unmask_and_level_IO_APIC_irq(cfg);
  2272. spin_unlock(&ioapic_lock);
  2273. }
  2274. #endif
  2275. }
  2276. static struct irq_chip ioapic_chip __read_mostly = {
  2277. .name = "IO-APIC",
  2278. .startup = startup_ioapic_irq,
  2279. .mask = mask_IO_APIC_irq,
  2280. .unmask = unmask_IO_APIC_irq,
  2281. .ack = ack_apic_edge,
  2282. .eoi = ack_apic_level,
  2283. #ifdef CONFIG_SMP
  2284. .set_affinity = set_ioapic_affinity_irq,
  2285. #endif
  2286. .retrigger = ioapic_retrigger_irq,
  2287. };
  2288. #ifdef CONFIG_INTR_REMAP
  2289. static struct irq_chip ir_ioapic_chip __read_mostly = {
  2290. .name = "IR-IO-APIC",
  2291. .startup = startup_ioapic_irq,
  2292. .mask = mask_IO_APIC_irq,
  2293. .unmask = unmask_IO_APIC_irq,
  2294. .ack = ack_x2apic_edge,
  2295. .eoi = ack_x2apic_level,
  2296. #ifdef CONFIG_SMP
  2297. .set_affinity = set_ir_ioapic_affinity_irq,
  2298. #endif
  2299. .retrigger = ioapic_retrigger_irq,
  2300. };
  2301. #endif
  2302. static inline void init_IO_APIC_traps(void)
  2303. {
  2304. int irq;
  2305. struct irq_desc *desc;
  2306. struct irq_cfg *cfg;
  2307. /*
  2308. * NOTE! The local APIC isn't very good at handling
  2309. * multiple interrupts at the same interrupt level.
  2310. * As the interrupt level is determined by taking the
  2311. * vector number and shifting that right by 4, we
  2312. * want to spread these out a bit so that they don't
  2313. * all fall in the same interrupt level.
  2314. *
  2315. * Also, we've got to be careful not to trash gate
  2316. * 0x80, because int 0x80 is hm, kind of importantish. ;)
  2317. */
  2318. for_each_irq_desc(irq, desc) {
  2319. if (!desc)
  2320. continue;
  2321. cfg = desc->chip_data;
  2322. if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
  2323. /*
  2324. * Hmm.. We don't have an entry for this,
  2325. * so default to an old-fashioned 8259
  2326. * interrupt if we can..
  2327. */
  2328. if (irq < NR_IRQS_LEGACY)
  2329. make_8259A_irq(irq);
  2330. else
  2331. /* Strange. Oh, well.. */
  2332. desc->chip = &no_irq_chip;
  2333. }
  2334. }
  2335. }
  2336. /*
  2337. * The local APIC irq-chip implementation:
  2338. */
  2339. static void mask_lapic_irq(unsigned int irq)
  2340. {
  2341. unsigned long v;
  2342. v = apic_read(APIC_LVT0);
  2343. apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
  2344. }
  2345. static void unmask_lapic_irq(unsigned int irq)
  2346. {
  2347. unsigned long v;
  2348. v = apic_read(APIC_LVT0);
  2349. apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
  2350. }
  2351. static void ack_lapic_irq(unsigned int irq)
  2352. {
  2353. ack_APIC_irq();
  2354. }
  2355. static struct irq_chip lapic_chip __read_mostly = {
  2356. .name = "local-APIC",
  2357. .mask = mask_lapic_irq,
  2358. .unmask = unmask_lapic_irq,
  2359. .ack = ack_lapic_irq,
  2360. };
  2361. static void lapic_register_intr(int irq, struct irq_desc *desc)
  2362. {
  2363. desc->status &= ~IRQ_LEVEL;
  2364. set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
  2365. "edge");
  2366. }
  2367. static void __init setup_nmi(void)
  2368. {
  2369. /*
  2370. * Dirty trick to enable the NMI watchdog ...
  2371. * We put the 8259A master into AEOI mode and
  2372. * unmask on all local APICs LVT0 as NMI.
  2373. *
  2374. * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
  2375. * is from Maciej W. Rozycki - so we do not have to EOI from
  2376. * the NMI handler or the timer interrupt.
  2377. */
  2378. apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
  2379. enable_NMI_through_LVT0();
  2380. apic_printk(APIC_VERBOSE, " done.\n");
  2381. }
  2382. /*
  2383. * This looks a bit hackish but it's about the only one way of sending
  2384. * a few INTA cycles to 8259As and any associated glue logic. ICR does
  2385. * not support the ExtINT mode, unfortunately. We need to send these
  2386. * cycles as some i82489DX-based boards have glue logic that keeps the
  2387. * 8259A interrupt line asserted until INTA. --macro
  2388. */
  2389. static inline void __init unlock_ExtINT_logic(void)
  2390. {
  2391. int apic, pin, i;
  2392. struct IO_APIC_route_entry entry0, entry1;
  2393. unsigned char save_control, save_freq_select;
  2394. pin = find_isa_irq_pin(8, mp_INT);
  2395. if (pin == -1) {
  2396. WARN_ON_ONCE(1);
  2397. return;
  2398. }
  2399. apic = find_isa_irq_apic(8, mp_INT);
  2400. if (apic == -1) {
  2401. WARN_ON_ONCE(1);
  2402. return;
  2403. }
  2404. entry0 = ioapic_read_entry(apic, pin);
  2405. clear_IO_APIC_pin(apic, pin);
  2406. memset(&entry1, 0, sizeof(entry1));
  2407. entry1.dest_mode = 0; /* physical delivery */
  2408. entry1.mask = 0; /* unmask IRQ now */
  2409. entry1.dest = hard_smp_processor_id();
  2410. entry1.delivery_mode = dest_ExtINT;
  2411. entry1.polarity = entry0.polarity;
  2412. entry1.trigger = 0;
  2413. entry1.vector = 0;
  2414. ioapic_write_entry(apic, pin, entry1);
  2415. save_control = CMOS_READ(RTC_CONTROL);
  2416. save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
  2417. CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
  2418. RTC_FREQ_SELECT);
  2419. CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
  2420. i = 100;
  2421. while (i-- > 0) {
  2422. mdelay(10);
  2423. if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
  2424. i -= 10;
  2425. }
  2426. CMOS_WRITE(save_control, RTC_CONTROL);
  2427. CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
  2428. clear_IO_APIC_pin(apic, pin);
  2429. ioapic_write_entry(apic, pin, entry0);
  2430. }
  2431. static int disable_timer_pin_1 __initdata;
  2432. /* Actually the next is obsolete, but keep it for paranoid reasons -AK */
  2433. static int __init disable_timer_pin_setup(char *arg)
  2434. {
  2435. disable_timer_pin_1 = 1;
  2436. return 0;
  2437. }
  2438. early_param("disable_timer_pin_1", disable_timer_pin_setup);
  2439. int timer_through_8259 __initdata;
  2440. /*
  2441. * This code may look a bit paranoid, but it's supposed to cooperate with
  2442. * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
  2443. * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
  2444. * fanatically on his truly buggy board.
  2445. *
  2446. * FIXME: really need to revamp this for all platforms.
  2447. */
  2448. static inline void __init check_timer(void)
  2449. {
  2450. struct irq_desc *desc = irq_to_desc(0);
  2451. struct irq_cfg *cfg = desc->chip_data;
  2452. int cpu = boot_cpu_id;
  2453. int apic1, pin1, apic2, pin2;
  2454. unsigned long flags;
  2455. unsigned int ver;
  2456. int no_pin1 = 0;
  2457. local_irq_save(flags);
  2458. ver = apic_read(APIC_LVR);
  2459. ver = GET_APIC_VERSION(ver);
  2460. /*
  2461. * get/set the timer IRQ vector:
  2462. */
  2463. disable_8259A_irq(0);
  2464. assign_irq_vector(0, cfg, TARGET_CPUS);
  2465. /*
  2466. * As IRQ0 is to be enabled in the 8259A, the virtual
  2467. * wire has to be disabled in the local APIC. Also
  2468. * timer interrupts need to be acknowledged manually in
  2469. * the 8259A for the i82489DX when using the NMI
  2470. * watchdog as that APIC treats NMIs as level-triggered.
  2471. * The AEOI mode will finish them in the 8259A
  2472. * automatically.
  2473. */
  2474. apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
  2475. init_8259A(1);
  2476. #ifdef CONFIG_X86_32
  2477. timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
  2478. #endif
  2479. pin1 = find_isa_irq_pin(0, mp_INT);
  2480. apic1 = find_isa_irq_apic(0, mp_INT);
  2481. pin2 = ioapic_i8259.pin;
  2482. apic2 = ioapic_i8259.apic;
  2483. apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
  2484. "apic1=%d pin1=%d apic2=%d pin2=%d\n",
  2485. cfg->vector, apic1, pin1, apic2, pin2);
  2486. /*
  2487. * Some BIOS writers are clueless and report the ExtINTA
  2488. * I/O APIC input from the cascaded 8259A as the timer
  2489. * interrupt input. So just in case, if only one pin
  2490. * was found above, try it both directly and through the
  2491. * 8259A.
  2492. */
  2493. if (pin1 == -1) {
  2494. #ifdef CONFIG_INTR_REMAP
  2495. if (intr_remapping_enabled)
  2496. panic("BIOS bug: timer not connected to IO-APIC");
  2497. #endif
  2498. pin1 = pin2;
  2499. apic1 = apic2;
  2500. no_pin1 = 1;
  2501. } else if (pin2 == -1) {
  2502. pin2 = pin1;
  2503. apic2 = apic1;
  2504. }
  2505. if (pin1 != -1) {
  2506. /*
  2507. * Ok, does IRQ0 through the IOAPIC work?
  2508. */
  2509. if (no_pin1) {
  2510. add_pin_to_irq_cpu(cfg, cpu, apic1, pin1);
  2511. setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
  2512. }
  2513. unmask_IO_APIC_irq_desc(desc);
  2514. if (timer_irq_works()) {
  2515. if (nmi_watchdog == NMI_IO_APIC) {
  2516. setup_nmi();
  2517. enable_8259A_irq(0);
  2518. }
  2519. if (disable_timer_pin_1 > 0)
  2520. clear_IO_APIC_pin(0, pin1);
  2521. goto out;
  2522. }
  2523. #ifdef CONFIG_INTR_REMAP
  2524. if (intr_remapping_enabled)
  2525. panic("timer doesn't work through Interrupt-remapped IO-APIC");
  2526. #endif
  2527. clear_IO_APIC_pin(apic1, pin1);
  2528. if (!no_pin1)
  2529. apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
  2530. "8254 timer not connected to IO-APIC\n");
  2531. apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
  2532. "(IRQ0) through the 8259A ...\n");
  2533. apic_printk(APIC_QUIET, KERN_INFO
  2534. "..... (found apic %d pin %d) ...\n", apic2, pin2);
  2535. /*
  2536. * legacy devices should be connected to IO APIC #0
  2537. */
  2538. replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2);
  2539. setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
  2540. unmask_IO_APIC_irq_desc(desc);
  2541. enable_8259A_irq(0);
  2542. if (timer_irq_works()) {
  2543. apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
  2544. timer_through_8259 = 1;
  2545. if (nmi_watchdog == NMI_IO_APIC) {
  2546. disable_8259A_irq(0);
  2547. setup_nmi();
  2548. enable_8259A_irq(0);
  2549. }
  2550. goto out;
  2551. }
  2552. /*
  2553. * Cleanup, just in case ...
  2554. */
  2555. disable_8259A_irq(0);
  2556. clear_IO_APIC_pin(apic2, pin2);
  2557. apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
  2558. }
  2559. if (nmi_watchdog == NMI_IO_APIC) {
  2560. apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
  2561. "through the IO-APIC - disabling NMI Watchdog!\n");
  2562. nmi_watchdog = NMI_NONE;
  2563. }
  2564. #ifdef CONFIG_X86_32
  2565. timer_ack = 0;
  2566. #endif
  2567. apic_printk(APIC_QUIET, KERN_INFO
  2568. "...trying to set up timer as Virtual Wire IRQ...\n");
  2569. lapic_register_intr(0, desc);
  2570. apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
  2571. enable_8259A_irq(0);
  2572. if (timer_irq_works()) {
  2573. apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
  2574. goto out;
  2575. }
  2576. disable_8259A_irq(0);
  2577. apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
  2578. apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
  2579. apic_printk(APIC_QUIET, KERN_INFO
  2580. "...trying to set up timer as ExtINT IRQ...\n");
  2581. init_8259A(0);
  2582. make_8259A_irq(0);
  2583. apic_write(APIC_LVT0, APIC_DM_EXTINT);
  2584. unlock_ExtINT_logic();
  2585. if (timer_irq_works()) {
  2586. apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
  2587. goto out;
  2588. }
  2589. apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
  2590. panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
  2591. "report. Then try booting with the 'noapic' option.\n");
  2592. out:
  2593. local_irq_restore(flags);
  2594. }
  2595. /*
  2596. * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
  2597. * to devices. However there may be an I/O APIC pin available for
  2598. * this interrupt regardless. The pin may be left unconnected, but
  2599. * typically it will be reused as an ExtINT cascade interrupt for
  2600. * the master 8259A. In the MPS case such a pin will normally be
  2601. * reported as an ExtINT interrupt in the MP table. With ACPI
  2602. * there is no provision for ExtINT interrupts, and in the absence
  2603. * of an override it would be treated as an ordinary ISA I/O APIC
  2604. * interrupt, that is edge-triggered and unmasked by default. We
  2605. * used to do this, but it caused problems on some systems because
  2606. * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
  2607. * the same ExtINT cascade interrupt to drive the local APIC of the
  2608. * bootstrap processor. Therefore we refrain from routing IRQ2 to
  2609. * the I/O APIC in all cases now. No actual device should request
  2610. * it anyway. --macro
  2611. */
  2612. #define PIC_IRQS (1 << PIC_CASCADE_IR)
  2613. void __init setup_IO_APIC(void)
  2614. {
  2615. #ifdef CONFIG_X86_32
  2616. enable_IO_APIC();
  2617. #else
  2618. /*
  2619. * calling enable_IO_APIC() is moved to setup_local_APIC for BP
  2620. */
  2621. #endif
  2622. io_apic_irqs = ~PIC_IRQS;
  2623. apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
  2624. /*
  2625. * Set up IO-APIC IRQ routing.
  2626. */
  2627. #ifdef CONFIG_X86_32
  2628. if (!acpi_ioapic)
  2629. setup_ioapic_ids_from_mpc();
  2630. #endif
  2631. sync_Arb_IDs();
  2632. setup_IO_APIC_irqs();
  2633. init_IO_APIC_traps();
  2634. check_timer();
  2635. }
  2636. /*
  2637. * Called after all the initialization is done. If we didnt find any
  2638. * APIC bugs then we can allow the modify fast path
  2639. */
  2640. static int __init io_apic_bug_finalize(void)
  2641. {
  2642. if (sis_apic_bug == -1)
  2643. sis_apic_bug = 0;
  2644. return 0;
  2645. }
  2646. late_initcall(io_apic_bug_finalize);
  2647. struct sysfs_ioapic_data {
  2648. struct sys_device dev;
  2649. struct IO_APIC_route_entry entry[0];
  2650. };
  2651. static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
  2652. static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
  2653. {
  2654. struct IO_APIC_route_entry *entry;
  2655. struct sysfs_ioapic_data *data;
  2656. int i;
  2657. data = container_of(dev, struct sysfs_ioapic_data, dev);
  2658. entry = data->entry;
  2659. for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
  2660. *entry = ioapic_read_entry(dev->id, i);
  2661. return 0;
  2662. }
  2663. static int ioapic_resume(struct sys_device *dev)
  2664. {
  2665. struct IO_APIC_route_entry *entry;
  2666. struct sysfs_ioapic_data *data;
  2667. unsigned long flags;
  2668. union IO_APIC_reg_00 reg_00;
  2669. int i;
  2670. data = container_of(dev, struct sysfs_ioapic_data, dev);
  2671. entry = data->entry;
  2672. spin_lock_irqsave(&ioapic_lock, flags);
  2673. reg_00.raw = io_apic_read(dev->id, 0);
  2674. if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
  2675. reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
  2676. io_apic_write(dev->id, 0, reg_00.raw);
  2677. }
  2678. spin_unlock_irqrestore(&ioapic_lock, flags);
  2679. for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
  2680. ioapic_write_entry(dev->id, i, entry[i]);
  2681. return 0;
  2682. }
  2683. static struct sysdev_class ioapic_sysdev_class = {
  2684. .name = "ioapic",
  2685. .suspend = ioapic_suspend,
  2686. .resume = ioapic_resume,
  2687. };
  2688. static int __init ioapic_init_sysfs(void)
  2689. {
  2690. struct sys_device * dev;
  2691. int i, size, error;
  2692. error = sysdev_class_register(&ioapic_sysdev_class);
  2693. if (error)
  2694. return error;
  2695. for (i = 0; i < nr_ioapics; i++ ) {
  2696. size = sizeof(struct sys_device) + nr_ioapic_registers[i]
  2697. * sizeof(struct IO_APIC_route_entry);
  2698. mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
  2699. if (!mp_ioapic_data[i]) {
  2700. printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
  2701. continue;
  2702. }
  2703. dev = &mp_ioapic_data[i]->dev;
  2704. dev->id = i;
  2705. dev->cls = &ioapic_sysdev_class;
  2706. error = sysdev_register(dev);
  2707. if (error) {
  2708. kfree(mp_ioapic_data[i]);
  2709. mp_ioapic_data[i] = NULL;
  2710. printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
  2711. continue;
  2712. }
  2713. }
  2714. return 0;
  2715. }
  2716. device_initcall(ioapic_init_sysfs);
  2717. /*
  2718. * Dynamic irq allocate and deallocation
  2719. */
  2720. unsigned int create_irq_nr(unsigned int irq_want)
  2721. {
  2722. /* Allocate an unused irq */
  2723. unsigned int irq;
  2724. unsigned int new;
  2725. unsigned long flags;
  2726. struct irq_cfg *cfg_new = NULL;
  2727. int cpu = boot_cpu_id;
  2728. struct irq_desc *desc_new = NULL;
  2729. irq = 0;
  2730. spin_lock_irqsave(&vector_lock, flags);
  2731. for (new = irq_want; new < NR_IRQS; new++) {
  2732. if (platform_legacy_irq(new))
  2733. continue;
  2734. desc_new = irq_to_desc_alloc_cpu(new, cpu);
  2735. if (!desc_new) {
  2736. printk(KERN_INFO "can not get irq_desc for %d\n", new);
  2737. continue;
  2738. }
  2739. cfg_new = desc_new->chip_data;
  2740. if (cfg_new->vector != 0)
  2741. continue;
  2742. if (__assign_irq_vector(new, cfg_new, TARGET_CPUS) == 0)
  2743. irq = new;
  2744. break;
  2745. }
  2746. spin_unlock_irqrestore(&vector_lock, flags);
  2747. if (irq > 0) {
  2748. dynamic_irq_init(irq);
  2749. /* restore it, in case dynamic_irq_init clear it */
  2750. if (desc_new)
  2751. desc_new->chip_data = cfg_new;
  2752. }
  2753. return irq;
  2754. }
  2755. static int nr_irqs_gsi = NR_IRQS_LEGACY;
  2756. int create_irq(void)
  2757. {
  2758. unsigned int irq_want;
  2759. int irq;
  2760. irq_want = nr_irqs_gsi;
  2761. irq = create_irq_nr(irq_want);
  2762. if (irq == 0)
  2763. irq = -1;
  2764. return irq;
  2765. }
  2766. void destroy_irq(unsigned int irq)
  2767. {
  2768. unsigned long flags;
  2769. struct irq_cfg *cfg;
  2770. struct irq_desc *desc;
  2771. /* store it, in case dynamic_irq_cleanup clear it */
  2772. desc = irq_to_desc(irq);
  2773. cfg = desc->chip_data;
  2774. dynamic_irq_cleanup(irq);
  2775. /* connect back irq_cfg */
  2776. if (desc)
  2777. desc->chip_data = cfg;
  2778. #ifdef CONFIG_INTR_REMAP
  2779. free_irte(irq);
  2780. #endif
  2781. spin_lock_irqsave(&vector_lock, flags);
  2782. __clear_irq_vector(irq, cfg);
  2783. spin_unlock_irqrestore(&vector_lock, flags);
  2784. }
  2785. /*
  2786. * MSI message composition
  2787. */
  2788. #ifdef CONFIG_PCI_MSI
  2789. static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
  2790. {
  2791. struct irq_cfg *cfg;
  2792. int err;
  2793. unsigned dest;
  2794. cfg = irq_cfg(irq);
  2795. err = assign_irq_vector(irq, cfg, TARGET_CPUS);
  2796. if (err)
  2797. return err;
  2798. dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
  2799. #ifdef CONFIG_INTR_REMAP
  2800. if (irq_remapped(irq)) {
  2801. struct irte irte;
  2802. int ir_index;
  2803. u16 sub_handle;
  2804. ir_index = map_irq_to_irte_handle(irq, &sub_handle);
  2805. BUG_ON(ir_index == -1);
  2806. memset (&irte, 0, sizeof(irte));
  2807. irte.present = 1;
  2808. irte.dst_mode = INT_DEST_MODE;
  2809. irte.trigger_mode = 0; /* edge */
  2810. irte.dlvry_mode = INT_DELIVERY_MODE;
  2811. irte.vector = cfg->vector;
  2812. irte.dest_id = IRTE_DEST(dest);
  2813. modify_irte(irq, &irte);
  2814. msg->address_hi = MSI_ADDR_BASE_HI;
  2815. msg->data = sub_handle;
  2816. msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
  2817. MSI_ADDR_IR_SHV |
  2818. MSI_ADDR_IR_INDEX1(ir_index) |
  2819. MSI_ADDR_IR_INDEX2(ir_index);
  2820. } else
  2821. #endif
  2822. {
  2823. msg->address_hi = MSI_ADDR_BASE_HI;
  2824. msg->address_lo =
  2825. MSI_ADDR_BASE_LO |
  2826. ((INT_DEST_MODE == 0) ?
  2827. MSI_ADDR_DEST_MODE_PHYSICAL:
  2828. MSI_ADDR_DEST_MODE_LOGICAL) |
  2829. ((INT_DELIVERY_MODE != dest_LowestPrio) ?
  2830. MSI_ADDR_REDIRECTION_CPU:
  2831. MSI_ADDR_REDIRECTION_LOWPRI) |
  2832. MSI_ADDR_DEST_ID(dest);
  2833. msg->data =
  2834. MSI_DATA_TRIGGER_EDGE |
  2835. MSI_DATA_LEVEL_ASSERT |
  2836. ((INT_DELIVERY_MODE != dest_LowestPrio) ?
  2837. MSI_DATA_DELIVERY_FIXED:
  2838. MSI_DATA_DELIVERY_LOWPRI) |
  2839. MSI_DATA_VECTOR(cfg->vector);
  2840. }
  2841. return err;
  2842. }
  2843. #ifdef CONFIG_SMP
  2844. static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
  2845. {
  2846. struct irq_desc *desc = irq_to_desc(irq);
  2847. struct irq_cfg *cfg;
  2848. struct msi_msg msg;
  2849. unsigned int dest;
  2850. dest = set_desc_affinity(desc, mask);
  2851. if (dest == BAD_APICID)
  2852. return;
  2853. cfg = desc->chip_data;
  2854. read_msi_msg_desc(desc, &msg);
  2855. msg.data &= ~MSI_DATA_VECTOR_MASK;
  2856. msg.data |= MSI_DATA_VECTOR(cfg->vector);
  2857. msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
  2858. msg.address_lo |= MSI_ADDR_DEST_ID(dest);
  2859. write_msi_msg_desc(desc, &msg);
  2860. }
  2861. #ifdef CONFIG_INTR_REMAP
  2862. /*
  2863. * Migrate the MSI irq to another cpumask. This migration is
  2864. * done in the process context using interrupt-remapping hardware.
  2865. */
  2866. static void
  2867. ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
  2868. {
  2869. struct irq_desc *desc = irq_to_desc(irq);
  2870. struct irq_cfg *cfg;
  2871. unsigned int dest;
  2872. struct irte irte;
  2873. if (get_irte(irq, &irte))
  2874. return;
  2875. dest = set_desc_affinity(desc, mask);
  2876. if (dest == BAD_APICID)
  2877. return;
  2878. irte.vector = cfg->vector;
  2879. irte.dest_id = IRTE_DEST(dest);
  2880. /*
  2881. * atomically update the IRTE with the new destination and vector.
  2882. */
  2883. modify_irte(irq, &irte);
  2884. /*
  2885. * After this point, all the interrupts will start arriving
  2886. * at the new destination. So, time to cleanup the previous
  2887. * vector allocation.
  2888. */
  2889. if (cfg->move_in_progress)
  2890. send_cleanup_vector(cfg);
  2891. }
  2892. #endif
  2893. #endif /* CONFIG_SMP */
  2894. /*
  2895. * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
  2896. * which implement the MSI or MSI-X Capability Structure.
  2897. */
  2898. static struct irq_chip msi_chip = {
  2899. .name = "PCI-MSI",
  2900. .unmask = unmask_msi_irq,
  2901. .mask = mask_msi_irq,
  2902. .ack = ack_apic_edge,
  2903. #ifdef CONFIG_SMP
  2904. .set_affinity = set_msi_irq_affinity,
  2905. #endif
  2906. .retrigger = ioapic_retrigger_irq,
  2907. };
  2908. #ifdef CONFIG_INTR_REMAP
  2909. static struct irq_chip msi_ir_chip = {
  2910. .name = "IR-PCI-MSI",
  2911. .unmask = unmask_msi_irq,
  2912. .mask = mask_msi_irq,
  2913. .ack = ack_x2apic_edge,
  2914. #ifdef CONFIG_SMP
  2915. .set_affinity = ir_set_msi_irq_affinity,
  2916. #endif
  2917. .retrigger = ioapic_retrigger_irq,
  2918. };
  2919. /*
  2920. * Map the PCI dev to the corresponding remapping hardware unit
  2921. * and allocate 'nvec' consecutive interrupt-remapping table entries
  2922. * in it.
  2923. */
  2924. static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
  2925. {
  2926. struct intel_iommu *iommu;
  2927. int index;
  2928. iommu = map_dev_to_ir(dev);
  2929. if (!iommu) {
  2930. printk(KERN_ERR
  2931. "Unable to map PCI %s to iommu\n", pci_name(dev));
  2932. return -ENOENT;
  2933. }
  2934. index = alloc_irte(iommu, irq, nvec);
  2935. if (index < 0) {
  2936. printk(KERN_ERR
  2937. "Unable to allocate %d IRTE for PCI %s\n", nvec,
  2938. pci_name(dev));
  2939. return -ENOSPC;
  2940. }
  2941. return index;
  2942. }
  2943. #endif
  2944. static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
  2945. {
  2946. int ret;
  2947. struct msi_msg msg;
  2948. ret = msi_compose_msg(dev, irq, &msg);
  2949. if (ret < 0)
  2950. return ret;
  2951. set_irq_msi(irq, msidesc);
  2952. write_msi_msg(irq, &msg);
  2953. #ifdef CONFIG_INTR_REMAP
  2954. if (irq_remapped(irq)) {
  2955. struct irq_desc *desc = irq_to_desc(irq);
  2956. /*
  2957. * irq migration in process context
  2958. */
  2959. desc->status |= IRQ_MOVE_PCNTXT;
  2960. set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
  2961. } else
  2962. #endif
  2963. set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
  2964. dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
  2965. return 0;
  2966. }
  2967. int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc)
  2968. {
  2969. unsigned int irq;
  2970. int ret;
  2971. unsigned int irq_want;
  2972. irq_want = nr_irqs_gsi;
  2973. irq = create_irq_nr(irq_want);
  2974. if (irq == 0)
  2975. return -1;
  2976. #ifdef CONFIG_INTR_REMAP
  2977. if (!intr_remapping_enabled)
  2978. goto no_ir;
  2979. ret = msi_alloc_irte(dev, irq, 1);
  2980. if (ret < 0)
  2981. goto error;
  2982. no_ir:
  2983. #endif
  2984. ret = setup_msi_irq(dev, msidesc, irq);
  2985. if (ret < 0) {
  2986. destroy_irq(irq);
  2987. return ret;
  2988. }
  2989. return 0;
  2990. #ifdef CONFIG_INTR_REMAP
  2991. error:
  2992. destroy_irq(irq);
  2993. return ret;
  2994. #endif
  2995. }
  2996. int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
  2997. {
  2998. unsigned int irq;
  2999. int ret, sub_handle;
  3000. struct msi_desc *msidesc;
  3001. unsigned int irq_want;
  3002. #ifdef CONFIG_INTR_REMAP
  3003. struct intel_iommu *iommu = 0;
  3004. int index = 0;
  3005. #endif
  3006. irq_want = nr_irqs_gsi;
  3007. sub_handle = 0;
  3008. list_for_each_entry(msidesc, &dev->msi_list, list) {
  3009. irq = create_irq_nr(irq_want);
  3010. irq_want++;
  3011. if (irq == 0)
  3012. return -1;
  3013. #ifdef CONFIG_INTR_REMAP
  3014. if (!intr_remapping_enabled)
  3015. goto no_ir;
  3016. if (!sub_handle) {
  3017. /*
  3018. * allocate the consecutive block of IRTE's
  3019. * for 'nvec'
  3020. */
  3021. index = msi_alloc_irte(dev, irq, nvec);
  3022. if (index < 0) {
  3023. ret = index;
  3024. goto error;
  3025. }
  3026. } else {
  3027. iommu = map_dev_to_ir(dev);
  3028. if (!iommu) {
  3029. ret = -ENOENT;
  3030. goto error;
  3031. }
  3032. /*
  3033. * setup the mapping between the irq and the IRTE
  3034. * base index, the sub_handle pointing to the
  3035. * appropriate interrupt remap table entry.
  3036. */
  3037. set_irte_irq(irq, iommu, index, sub_handle);
  3038. }
  3039. no_ir:
  3040. #endif
  3041. ret = setup_msi_irq(dev, msidesc, irq);
  3042. if (ret < 0)
  3043. goto error;
  3044. sub_handle++;
  3045. }
  3046. return 0;
  3047. error:
  3048. destroy_irq(irq);
  3049. return ret;
  3050. }
  3051. void arch_teardown_msi_irq(unsigned int irq)
  3052. {
  3053. destroy_irq(irq);
  3054. }
  3055. #ifdef CONFIG_DMAR
  3056. #ifdef CONFIG_SMP
  3057. static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
  3058. {
  3059. struct irq_desc *desc = irq_to_desc(irq);
  3060. struct irq_cfg *cfg;
  3061. struct msi_msg msg;
  3062. unsigned int dest;
  3063. dest = set_desc_affinity(desc, mask);
  3064. if (dest == BAD_APICID)
  3065. return;
  3066. cfg = desc->chip_data;
  3067. dmar_msi_read(irq, &msg);
  3068. msg.data &= ~MSI_DATA_VECTOR_MASK;
  3069. msg.data |= MSI_DATA_VECTOR(cfg->vector);
  3070. msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
  3071. msg.address_lo |= MSI_ADDR_DEST_ID(dest);
  3072. dmar_msi_write(irq, &msg);
  3073. }
  3074. #endif /* CONFIG_SMP */
  3075. struct irq_chip dmar_msi_type = {
  3076. .name = "DMAR_MSI",
  3077. .unmask = dmar_msi_unmask,
  3078. .mask = dmar_msi_mask,
  3079. .ack = ack_apic_edge,
  3080. #ifdef CONFIG_SMP
  3081. .set_affinity = dmar_msi_set_affinity,
  3082. #endif
  3083. .retrigger = ioapic_retrigger_irq,
  3084. };
  3085. int arch_setup_dmar_msi(unsigned int irq)
  3086. {
  3087. int ret;
  3088. struct msi_msg msg;
  3089. ret = msi_compose_msg(NULL, irq, &msg);
  3090. if (ret < 0)
  3091. return ret;
  3092. dmar_msi_write(irq, &msg);
  3093. set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
  3094. "edge");
  3095. return 0;
  3096. }
  3097. #endif
  3098. #ifdef CONFIG_HPET_TIMER
  3099. #ifdef CONFIG_SMP
  3100. static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
  3101. {
  3102. struct irq_desc *desc = irq_to_desc(irq);
  3103. struct irq_cfg *cfg;
  3104. struct msi_msg msg;
  3105. unsigned int dest;
  3106. dest = set_desc_affinity(desc, mask);
  3107. if (dest == BAD_APICID)
  3108. return;
  3109. cfg = desc->chip_data;
  3110. hpet_msi_read(irq, &msg);
  3111. msg.data &= ~MSI_DATA_VECTOR_MASK;
  3112. msg.data |= MSI_DATA_VECTOR(cfg->vector);
  3113. msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
  3114. msg.address_lo |= MSI_ADDR_DEST_ID(dest);
  3115. hpet_msi_write(irq, &msg);
  3116. }
  3117. #endif /* CONFIG_SMP */
  3118. struct irq_chip hpet_msi_type = {
  3119. .name = "HPET_MSI",
  3120. .unmask = hpet_msi_unmask,
  3121. .mask = hpet_msi_mask,
  3122. .ack = ack_apic_edge,
  3123. #ifdef CONFIG_SMP
  3124. .set_affinity = hpet_msi_set_affinity,
  3125. #endif
  3126. .retrigger = ioapic_retrigger_irq,
  3127. };
  3128. int arch_setup_hpet_msi(unsigned int irq)
  3129. {
  3130. int ret;
  3131. struct msi_msg msg;
  3132. ret = msi_compose_msg(NULL, irq, &msg);
  3133. if (ret < 0)
  3134. return ret;
  3135. hpet_msi_write(irq, &msg);
  3136. set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
  3137. "edge");
  3138. return 0;
  3139. }
  3140. #endif
  3141. #endif /* CONFIG_PCI_MSI */
  3142. /*
  3143. * Hypertransport interrupt support
  3144. */
  3145. #ifdef CONFIG_HT_IRQ
  3146. #ifdef CONFIG_SMP
  3147. static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
  3148. {
  3149. struct ht_irq_msg msg;
  3150. fetch_ht_irq_msg(irq, &msg);
  3151. msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
  3152. msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
  3153. msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
  3154. msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
  3155. write_ht_irq_msg(irq, &msg);
  3156. }
  3157. static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
  3158. {
  3159. struct irq_desc *desc = irq_to_desc(irq);
  3160. struct irq_cfg *cfg;
  3161. unsigned int dest;
  3162. dest = set_desc_affinity(desc, mask);
  3163. if (dest == BAD_APICID)
  3164. return;
  3165. cfg = desc->chip_data;
  3166. target_ht_irq(irq, dest, cfg->vector);
  3167. }
  3168. #endif
  3169. static struct irq_chip ht_irq_chip = {
  3170. .name = "PCI-HT",
  3171. .mask = mask_ht_irq,
  3172. .unmask = unmask_ht_irq,
  3173. .ack = ack_apic_edge,
  3174. #ifdef CONFIG_SMP
  3175. .set_affinity = set_ht_irq_affinity,
  3176. #endif
  3177. .retrigger = ioapic_retrigger_irq,
  3178. };
  3179. int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
  3180. {
  3181. struct irq_cfg *cfg;
  3182. int err;
  3183. cfg = irq_cfg(irq);
  3184. err = assign_irq_vector(irq, cfg, TARGET_CPUS);
  3185. if (!err) {
  3186. struct ht_irq_msg msg;
  3187. unsigned dest;
  3188. dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
  3189. msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
  3190. msg.address_lo =
  3191. HT_IRQ_LOW_BASE |
  3192. HT_IRQ_LOW_DEST_ID(dest) |
  3193. HT_IRQ_LOW_VECTOR(cfg->vector) |
  3194. ((INT_DEST_MODE == 0) ?
  3195. HT_IRQ_LOW_DM_PHYSICAL :
  3196. HT_IRQ_LOW_DM_LOGICAL) |
  3197. HT_IRQ_LOW_RQEOI_EDGE |
  3198. ((INT_DELIVERY_MODE != dest_LowestPrio) ?
  3199. HT_IRQ_LOW_MT_FIXED :
  3200. HT_IRQ_LOW_MT_ARBITRATED) |
  3201. HT_IRQ_LOW_IRQ_MASKED;
  3202. write_ht_irq_msg(irq, &msg);
  3203. set_irq_chip_and_handler_name(irq, &ht_irq_chip,
  3204. handle_edge_irq, "edge");
  3205. dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
  3206. }
  3207. return err;
  3208. }
  3209. #endif /* CONFIG_HT_IRQ */
  3210. #ifdef CONFIG_X86_64
  3211. /*
  3212. * Re-target the irq to the specified CPU and enable the specified MMR located
  3213. * on the specified blade to allow the sending of MSIs to the specified CPU.
  3214. */
  3215. int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
  3216. unsigned long mmr_offset)
  3217. {
  3218. const struct cpumask *eligible_cpu = cpumask_of(cpu);
  3219. struct irq_cfg *cfg;
  3220. int mmr_pnode;
  3221. unsigned long mmr_value;
  3222. struct uv_IO_APIC_route_entry *entry;
  3223. unsigned long flags;
  3224. int err;
  3225. cfg = irq_cfg(irq);
  3226. err = assign_irq_vector(irq, cfg, eligible_cpu);
  3227. if (err != 0)
  3228. return err;
  3229. spin_lock_irqsave(&vector_lock, flags);
  3230. set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
  3231. irq_name);
  3232. spin_unlock_irqrestore(&vector_lock, flags);
  3233. mmr_value = 0;
  3234. entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
  3235. BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
  3236. entry->vector = cfg->vector;
  3237. entry->delivery_mode = INT_DELIVERY_MODE;
  3238. entry->dest_mode = INT_DEST_MODE;
  3239. entry->polarity = 0;
  3240. entry->trigger = 0;
  3241. entry->mask = 0;
  3242. entry->dest = cpu_mask_to_apicid(eligible_cpu);
  3243. mmr_pnode = uv_blade_to_pnode(mmr_blade);
  3244. uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
  3245. return irq;
  3246. }
  3247. /*
  3248. * Disable the specified MMR located on the specified blade so that MSIs are
  3249. * longer allowed to be sent.
  3250. */
  3251. void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
  3252. {
  3253. unsigned long mmr_value;
  3254. struct uv_IO_APIC_route_entry *entry;
  3255. int mmr_pnode;
  3256. mmr_value = 0;
  3257. entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
  3258. BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
  3259. entry->mask = 1;
  3260. mmr_pnode = uv_blade_to_pnode(mmr_blade);
  3261. uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
  3262. }
  3263. #endif /* CONFIG_X86_64 */
  3264. int __init io_apic_get_redir_entries (int ioapic)
  3265. {
  3266. union IO_APIC_reg_01 reg_01;
  3267. unsigned long flags;
  3268. spin_lock_irqsave(&ioapic_lock, flags);
  3269. reg_01.raw = io_apic_read(ioapic, 1);
  3270. spin_unlock_irqrestore(&ioapic_lock, flags);
  3271. return reg_01.bits.entries;
  3272. }
  3273. void __init probe_nr_irqs_gsi(void)
  3274. {
  3275. int idx;
  3276. int nr = 0;
  3277. for (idx = 0; idx < nr_ioapics; idx++)
  3278. nr += io_apic_get_redir_entries(idx) + 1;
  3279. if (nr > nr_irqs_gsi)
  3280. nr_irqs_gsi = nr;
  3281. }
  3282. /* --------------------------------------------------------------------------
  3283. ACPI-based IOAPIC Configuration
  3284. -------------------------------------------------------------------------- */
  3285. #ifdef CONFIG_ACPI
  3286. #ifdef CONFIG_X86_32
  3287. int __init io_apic_get_unique_id(int ioapic, int apic_id)
  3288. {
  3289. union IO_APIC_reg_00 reg_00;
  3290. static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
  3291. physid_mask_t tmp;
  3292. unsigned long flags;
  3293. int i = 0;
  3294. /*
  3295. * The P4 platform supports up to 256 APIC IDs on two separate APIC
  3296. * buses (one for LAPICs, one for IOAPICs), where predecessors only
  3297. * supports up to 16 on one shared APIC bus.
  3298. *
  3299. * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
  3300. * advantage of new APIC bus architecture.
  3301. */
  3302. if (physids_empty(apic_id_map))
  3303. apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);
  3304. spin_lock_irqsave(&ioapic_lock, flags);
  3305. reg_00.raw = io_apic_read(ioapic, 0);
  3306. spin_unlock_irqrestore(&ioapic_lock, flags);
  3307. if (apic_id >= get_physical_broadcast()) {
  3308. printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
  3309. "%d\n", ioapic, apic_id, reg_00.bits.ID);
  3310. apic_id = reg_00.bits.ID;
  3311. }
  3312. /*
  3313. * Every APIC in a system must have a unique ID or we get lots of nice
  3314. * 'stuck on smp_invalidate_needed IPI wait' messages.
  3315. */
  3316. if (check_apicid_used(apic_id_map, apic_id)) {
  3317. for (i = 0; i < get_physical_broadcast(); i++) {
  3318. if (!check_apicid_used(apic_id_map, i))
  3319. break;
  3320. }
  3321. if (i == get_physical_broadcast())
  3322. panic("Max apic_id exceeded!\n");
  3323. printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
  3324. "trying %d\n", ioapic, apic_id, i);
  3325. apic_id = i;
  3326. }
  3327. tmp = apicid_to_cpu_present(apic_id);
  3328. physids_or(apic_id_map, apic_id_map, tmp);
  3329. if (reg_00.bits.ID != apic_id) {
  3330. reg_00.bits.ID = apic_id;
  3331. spin_lock_irqsave(&ioapic_lock, flags);
  3332. io_apic_write(ioapic, 0, reg_00.raw);
  3333. reg_00.raw = io_apic_read(ioapic, 0);
  3334. spin_unlock_irqrestore(&ioapic_lock, flags);
  3335. /* Sanity check */
  3336. if (reg_00.bits.ID != apic_id) {
  3337. printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
  3338. return -1;
  3339. }
  3340. }
  3341. apic_printk(APIC_VERBOSE, KERN_INFO
  3342. "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
  3343. return apic_id;
  3344. }
  3345. int __init io_apic_get_version(int ioapic)
  3346. {
  3347. union IO_APIC_reg_01 reg_01;
  3348. unsigned long flags;
  3349. spin_lock_irqsave(&ioapic_lock, flags);
  3350. reg_01.raw = io_apic_read(ioapic, 1);
  3351. spin_unlock_irqrestore(&ioapic_lock, flags);
  3352. return reg_01.bits.version;
  3353. }
  3354. #endif
  3355. int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
  3356. {
  3357. struct irq_desc *desc;
  3358. struct irq_cfg *cfg;
  3359. int cpu = boot_cpu_id;
  3360. if (!IO_APIC_IRQ(irq)) {
  3361. apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
  3362. ioapic);
  3363. return -EINVAL;
  3364. }
  3365. desc = irq_to_desc_alloc_cpu(irq, cpu);
  3366. if (!desc) {
  3367. printk(KERN_INFO "can not get irq_desc %d\n", irq);
  3368. return 0;
  3369. }
  3370. /*
  3371. * IRQs < 16 are already in the irq_2_pin[] map
  3372. */
  3373. if (irq >= NR_IRQS_LEGACY) {
  3374. cfg = desc->chip_data;
  3375. add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
  3376. }
  3377. setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);
  3378. return 0;
  3379. }
  3380. int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
  3381. {
  3382. int i;
  3383. if (skip_ioapic_setup)
  3384. return -1;
  3385. for (i = 0; i < mp_irq_entries; i++)
  3386. if (mp_irqs[i].mp_irqtype == mp_INT &&
  3387. mp_irqs[i].mp_srcbusirq == bus_irq)
  3388. break;
  3389. if (i >= mp_irq_entries)
  3390. return -1;
  3391. *trigger = irq_trigger(i);
  3392. *polarity = irq_polarity(i);
  3393. return 0;
  3394. }
  3395. #endif /* CONFIG_ACPI */
  3396. /*
  3397. * This function currently is only a helper for the i386 smp boot process where
  3398. * we need to reprogram the ioredtbls to cater for the cpus which have come online
  3399. * so mask in all cases should simply be TARGET_CPUS
  3400. */
  3401. #ifdef CONFIG_SMP
  3402. void __init setup_ioapic_dest(void)
  3403. {
  3404. int pin, ioapic, irq, irq_entry;
  3405. struct irq_desc *desc;
  3406. struct irq_cfg *cfg;
  3407. const struct cpumask *mask;
  3408. if (skip_ioapic_setup == 1)
  3409. return;
  3410. for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
  3411. for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
  3412. irq_entry = find_irq_entry(ioapic, pin, mp_INT);
  3413. if (irq_entry == -1)
  3414. continue;
  3415. irq = pin_2_irq(irq_entry, ioapic, pin);
  3416. /* setup_IO_APIC_irqs could fail to get vector for some device
  3417. * when you have too many devices, because at that time only boot
  3418. * cpu is online.
  3419. */
  3420. desc = irq_to_desc(irq);
  3421. cfg = desc->chip_data;
  3422. if (!cfg->vector) {
  3423. setup_IO_APIC_irq(ioapic, pin, irq, desc,
  3424. irq_trigger(irq_entry),
  3425. irq_polarity(irq_entry));
  3426. continue;
  3427. }
  3428. /*
  3429. * Honour affinities which have been set in early boot
  3430. */
  3431. if (desc->status &
  3432. (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
  3433. mask = &desc->affinity;
  3434. else
  3435. mask = TARGET_CPUS;
  3436. #ifdef CONFIG_INTR_REMAP
  3437. if (intr_remapping_enabled)
  3438. set_ir_ioapic_affinity_irq_desc(desc, mask);
  3439. else
  3440. #endif
  3441. set_ioapic_affinity_irq_desc(desc, mask);
  3442. }
  3443. }
  3444. }
  3445. #endif
  3446. #define IOAPIC_RESOURCE_NAME_SIZE 11
  3447. static struct resource *ioapic_resources;
  3448. static struct resource * __init ioapic_setup_resources(void)
  3449. {
  3450. unsigned long n;
  3451. struct resource *res;
  3452. char *mem;
  3453. int i;
  3454. if (nr_ioapics <= 0)
  3455. return NULL;
  3456. n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
  3457. n *= nr_ioapics;
  3458. mem = alloc_bootmem(n);
  3459. res = (void *)mem;
  3460. if (mem != NULL) {
  3461. mem += sizeof(struct resource) * nr_ioapics;
  3462. for (i = 0; i < nr_ioapics; i++) {
  3463. res[i].name = mem;
  3464. res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
  3465. sprintf(mem, "IOAPIC %u", i);
  3466. mem += IOAPIC_RESOURCE_NAME_SIZE;
  3467. }
  3468. }
  3469. ioapic_resources = res;
  3470. return res;
  3471. }
  3472. void __init ioapic_init_mappings(void)
  3473. {
  3474. unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
  3475. struct resource *ioapic_res;
  3476. int i;
  3477. ioapic_res = ioapic_setup_resources();
  3478. for (i = 0; i < nr_ioapics; i++) {
  3479. if (smp_found_config) {
  3480. ioapic_phys = mp_ioapics[i].mp_apicaddr;
  3481. #ifdef CONFIG_X86_32
  3482. if (!ioapic_phys) {
  3483. printk(KERN_ERR
  3484. "WARNING: bogus zero IO-APIC "
  3485. "address found in MPTABLE, "
  3486. "disabling IO/APIC support!\n");
  3487. smp_found_config = 0;
  3488. skip_ioapic_setup = 1;
  3489. goto fake_ioapic_page;
  3490. }
  3491. #endif
  3492. } else {
  3493. #ifdef CONFIG_X86_32
  3494. fake_ioapic_page:
  3495. #endif
  3496. ioapic_phys = (unsigned long)
  3497. alloc_bootmem_pages(PAGE_SIZE);
  3498. ioapic_phys = __pa(ioapic_phys);
  3499. }
  3500. set_fixmap_nocache(idx, ioapic_phys);
  3501. apic_printk(APIC_VERBOSE,
  3502. "mapped IOAPIC to %08lx (%08lx)\n",
  3503. __fix_to_virt(idx), ioapic_phys);
  3504. idx++;
  3505. if (ioapic_res != NULL) {
  3506. ioapic_res->start = ioapic_phys;
  3507. ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
  3508. ioapic_res++;
  3509. }
  3510. }
  3511. }
  3512. static int __init ioapic_insert_resources(void)
  3513. {
  3514. int i;
  3515. struct resource *r = ioapic_resources;
  3516. if (!r) {
  3517. printk(KERN_ERR
  3518. "IO APIC resources could be not be allocated.\n");
  3519. return -1;
  3520. }
  3521. for (i = 0; i < nr_ioapics; i++) {
  3522. insert_resource(&iomem_resource, r);
  3523. r++;
  3524. }
  3525. return 0;
  3526. }
  3527. /* Insert the IO APIC resources after PCI initialization has occured to handle
  3528. * IO APICS that are mapped in on a BAR in PCI space. */
  3529. late_initcall(ioapic_insert_resources);