iosapic.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176
  1. /*
  2. * I/O SAPIC support.
  3. *
  4. * Copyright (C) 1999 Intel Corp.
  5. * Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com>
  6. * Copyright (C) 2000-2002 J.I. Lee <jung-ik.lee@intel.com>
  7. * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co.
  8. * David Mosberger-Tang <davidm@hpl.hp.com>
  9. * Copyright (C) 1999 VA Linux Systems
  10. * Copyright (C) 1999,2000 Walt Drummond <drummond@valinux.com>
  11. *
  12. * 00/04/19 D. Mosberger Rewritten to mirror more closely the x86 I/O
  13. * APIC code. In particular, we now have separate
  14. * handlers for edge and level triggered
  15. * interrupts.
  16. * 00/10/27 Asit Mallick, Goutham Rao <goutham.rao@intel.com> IRQ vector
  17. * allocation PCI to vector mapping, shared PCI
  18. * interrupts.
  19. * 00/10/27 D. Mosberger Document things a bit more to make them more
  20. * understandable. Clean up much of the old
  21. * IOSAPIC cruft.
  22. * 01/07/27 J.I. Lee PCI irq routing, Platform/Legacy interrupts
  23. * and fixes for ACPI S5(SoftOff) support.
  24. * 02/01/23 J.I. Lee iosapic pgm fixes for PCI irq routing from _PRT
  25. * 02/01/07 E. Focht <efocht@ess.nec.de> Redirectable interrupt
  26. * vectors in iosapic_set_affinity(),
  27. * initializations for /proc/irq/#/smp_affinity
  28. * 02/04/02 P. Diefenbaugh Cleaned up ACPI PCI IRQ routing.
  29. * 02/04/18 J.I. Lee bug fix in iosapic_init_pci_irq
  30. * 02/04/30 J.I. Lee bug fix in find_iosapic to fix ACPI PCI IRQ to
  31. * IOSAPIC mapping error
  32. * 02/07/29 T. Kochi Allocate interrupt vectors dynamically
  33. * 02/08/04 T. Kochi Cleaned up terminology (irq, global system
  34. * interrupt, vector, etc.)
  35. * 02/09/20 D. Mosberger Simplified by taking advantage of ACPI's
  36. * pci_irq code.
  37. * 03/02/19 B. Helgaas Make pcat_compat system-wide, not per-IOSAPIC.
  38. * Remove iosapic_address & gsi_base from
  39. * external interfaces. Rationalize
  40. * __init/__devinit attributes.
  41. * 04/12/04 Ashok Raj <ashok.raj@intel.com> Intel Corporation 2004
  42. * Updated to work with irq migration necessary
  43. * for CPU Hotplug
  44. */
  45. /*
  46. * Here is what the interrupt logic between a PCI device and the kernel looks
  47. * like:
  48. *
  49. * (1) A PCI device raises one of the four interrupt pins (INTA, INTB, INTC,
  50. * INTD). The device is uniquely identified by its bus-, and slot-number
  51. * (the function number does not matter here because all functions share
  52. * the same interrupt lines).
  53. *
  54. * (2) The motherboard routes the interrupt line to a pin on a IOSAPIC
  55. * controller. Multiple interrupt lines may have to share the same
  56. * IOSAPIC pin (if they're level triggered and use the same polarity).
  57. * Each interrupt line has a unique Global System Interrupt (GSI) number
  58. * which can be calculated as the sum of the controller's base GSI number
  59. * and the IOSAPIC pin number to which the line connects.
  60. *
  61. * (3) The IOSAPIC uses an internal routing table entries (RTEs) to map the
  62. * IOSAPIC pin into the IA-64 interrupt vector. This interrupt vector is then
  63. * sent to the CPU.
  64. *
  65. * (4) The kernel recognizes an interrupt as an IRQ. The IRQ interface is
  66. * used as architecture-independent interrupt handling mechanism in Linux.
  67. * As an IRQ is a number, we have to have
  68. * IA-64 interrupt vector number <-> IRQ number mapping. On smaller
  69. * systems, we use one-to-one mapping between IA-64 vector and IRQ. A
  70. * platform can implement platform_irq_to_vector(irq) and
  71. * platform_local_vector_to_irq(vector) APIs to differentiate the mapping.
  72. * Please see also include/asm-ia64/hw_irq.h for those APIs.
  73. *
  74. * To sum up, there are three levels of mappings involved:
  75. *
  76. * PCI pin -> global system interrupt (GSI) -> IA-64 vector <-> IRQ
  77. *
  78. * Note: The term "IRQ" is loosely used everywhere in Linux kernel to
  79. * describeinterrupts. Now we use "IRQ" only for Linux IRQ's. ISA IRQ
  80. * (isa_irq) is the only exception in this source code.
  81. */
  82. #include <linux/acpi.h>
  83. #include <linux/init.h>
  84. #include <linux/irq.h>
  85. #include <linux/kernel.h>
  86. #include <linux/list.h>
  87. #include <linux/pci.h>
  88. #include <linux/smp.h>
  89. #include <linux/string.h>
  90. #include <linux/bootmem.h>
  91. #include <asm/delay.h>
  92. #include <asm/hw_irq.h>
  93. #include <asm/io.h>
  94. #include <asm/iosapic.h>
  95. #include <asm/machvec.h>
  96. #include <asm/processor.h>
  97. #include <asm/ptrace.h>
  98. #include <asm/system.h>
  99. #undef DEBUG_INTERRUPT_ROUTING
  100. #ifdef DEBUG_INTERRUPT_ROUTING
  101. #define DBG(fmt...) printk(fmt)
  102. #else
  103. #define DBG(fmt...)
  104. #endif
  105. #define NR_PREALLOCATE_RTE_ENTRIES \
  106. (PAGE_SIZE / sizeof(struct iosapic_rte_info))
  107. #define RTE_PREALLOCATED (1)
  108. static DEFINE_SPINLOCK(iosapic_lock);
  109. /*
  110. * These tables map IA-64 vectors to the IOSAPIC pin that generates this
  111. * vector.
  112. */
  113. struct iosapic_rte_info {
  114. struct list_head rte_list; /* node in list of RTEs sharing the
  115. * same vector */
  116. char __iomem *addr; /* base address of IOSAPIC */
  117. unsigned int gsi_base; /* first GSI assigned to this
  118. * IOSAPIC */
  119. char rte_index; /* IOSAPIC RTE index */
  120. int refcnt; /* reference counter */
  121. unsigned int flags; /* flags */
  122. } ____cacheline_aligned;
  123. static struct iosapic_intr_info {
  124. struct list_head rtes; /* RTEs using this vector (empty =>
  125. * not an IOSAPIC interrupt) */
  126. int count; /* # of RTEs that shares this vector */
  127. u32 low32; /* current value of low word of
  128. * Redirection table entry */
  129. unsigned int dest; /* destination CPU physical ID */
  130. unsigned char dmode : 3; /* delivery mode (see iosapic.h) */
  131. unsigned char polarity: 1; /* interrupt polarity
  132. * (see iosapic.h) */
  133. unsigned char trigger : 1; /* trigger mode (see iosapic.h) */
  134. } iosapic_intr_info[IA64_NUM_VECTORS];
  135. static struct iosapic {
  136. char __iomem *addr; /* base address of IOSAPIC */
  137. unsigned int gsi_base; /* first GSI assigned to this
  138. * IOSAPIC */
  139. unsigned short num_rte; /* # of RTEs on this IOSAPIC */
  140. int rtes_inuse; /* # of RTEs in use on this IOSAPIC */
  141. #ifdef CONFIG_NUMA
  142. unsigned short node; /* numa node association via pxm */
  143. #endif
  144. } iosapic_lists[NR_IOSAPICS];
  145. static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */
  146. static int iosapic_kmalloc_ok;
  147. static LIST_HEAD(free_rte_list);
  148. /*
  149. * Find an IOSAPIC associated with a GSI
  150. */
  151. static inline int
  152. find_iosapic (unsigned int gsi)
  153. {
  154. int i;
  155. for (i = 0; i < NR_IOSAPICS; i++) {
  156. if ((unsigned) (gsi - iosapic_lists[i].gsi_base) <
  157. iosapic_lists[i].num_rte)
  158. return i;
  159. }
  160. return -1;
  161. }
  162. static inline int
  163. _gsi_to_vector (unsigned int gsi)
  164. {
  165. struct iosapic_intr_info *info;
  166. struct iosapic_rte_info *rte;
  167. for (info = iosapic_intr_info; info <
  168. iosapic_intr_info + IA64_NUM_VECTORS; ++info)
  169. list_for_each_entry(rte, &info->rtes, rte_list)
  170. if (rte->gsi_base + rte->rte_index == gsi)
  171. return info - iosapic_intr_info;
  172. return -1;
  173. }
  174. /*
  175. * Translate GSI number to the corresponding IA-64 interrupt vector. If no
  176. * entry exists, return -1.
  177. */
  178. inline int
  179. gsi_to_vector (unsigned int gsi)
  180. {
  181. return _gsi_to_vector(gsi);
  182. }
  183. int
  184. gsi_to_irq (unsigned int gsi)
  185. {
  186. unsigned long flags;
  187. int irq;
  188. /*
  189. * XXX fix me: this assumes an identity mapping between IA-64 vector
  190. * and Linux irq numbers...
  191. */
  192. spin_lock_irqsave(&iosapic_lock, flags);
  193. irq = _gsi_to_vector(gsi);
  194. spin_unlock_irqrestore(&iosapic_lock, flags);
  195. return irq;
  196. }
  197. static struct iosapic_rte_info *gsi_vector_to_rte(unsigned int gsi,
  198. unsigned int vec)
  199. {
  200. struct iosapic_rte_info *rte;
  201. list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list)
  202. if (rte->gsi_base + rte->rte_index == gsi)
  203. return rte;
  204. return NULL;
  205. }
  206. static void
  207. set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask)
  208. {
  209. unsigned long pol, trigger, dmode;
  210. u32 low32, high32;
  211. char __iomem *addr;
  212. int rte_index;
  213. char redir;
  214. struct iosapic_rte_info *rte;
  215. DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest);
  216. rte = gsi_vector_to_rte(gsi, vector);
  217. if (!rte)
  218. return; /* not an IOSAPIC interrupt */
  219. rte_index = rte->rte_index;
  220. addr = rte->addr;
  221. pol = iosapic_intr_info[vector].polarity;
  222. trigger = iosapic_intr_info[vector].trigger;
  223. dmode = iosapic_intr_info[vector].dmode;
  224. redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
  225. #ifdef CONFIG_SMP
  226. {
  227. unsigned int irq;
  228. for (irq = 0; irq < NR_IRQS; ++irq)
  229. if (irq_to_vector(irq) == vector) {
  230. set_irq_affinity_info(irq,
  231. (int)(dest & 0xffff),
  232. redir);
  233. break;
  234. }
  235. }
  236. #endif
  237. low32 = ((pol << IOSAPIC_POLARITY_SHIFT) |
  238. (trigger << IOSAPIC_TRIGGER_SHIFT) |
  239. (dmode << IOSAPIC_DELIVERY_SHIFT) |
  240. ((mask ? 1 : 0) << IOSAPIC_MASK_SHIFT) |
  241. vector);
  242. /* dest contains both id and eid */
  243. high32 = (dest << IOSAPIC_DEST_SHIFT);
  244. iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32);
  245. iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
  246. iosapic_intr_info[vector].low32 = low32;
  247. iosapic_intr_info[vector].dest = dest;
  248. }
  249. static void
  250. nop (unsigned int irq)
  251. {
  252. /* do nothing... */
  253. }
  254. #ifdef CONFIG_KEXEC
  255. void
  256. kexec_disable_iosapic(void)
  257. {
  258. struct iosapic_intr_info *info;
  259. struct iosapic_rte_info *rte;
  260. u8 vec = 0;
  261. for (info = iosapic_intr_info; info <
  262. iosapic_intr_info + IA64_NUM_VECTORS; ++info, ++vec) {
  263. list_for_each_entry(rte, &info->rtes,
  264. rte_list) {
  265. iosapic_write(rte->addr,
  266. IOSAPIC_RTE_LOW(rte->rte_index),
  267. IOSAPIC_MASK|vec);
  268. iosapic_eoi(rte->addr, vec);
  269. }
  270. }
  271. }
  272. #endif
  273. static void
  274. mask_irq (unsigned int irq)
  275. {
  276. unsigned long flags;
  277. char __iomem *addr;
  278. u32 low32;
  279. int rte_index;
  280. ia64_vector vec = irq_to_vector(irq);
  281. struct iosapic_rte_info *rte;
  282. if (list_empty(&iosapic_intr_info[vec].rtes))
  283. return; /* not an IOSAPIC interrupt! */
  284. spin_lock_irqsave(&iosapic_lock, flags);
  285. /* set only the mask bit */
  286. low32 = iosapic_intr_info[vec].low32 |= IOSAPIC_MASK;
  287. list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) {
  288. addr = rte->addr;
  289. rte_index = rte->rte_index;
  290. iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
  291. }
  292. spin_unlock_irqrestore(&iosapic_lock, flags);
  293. }
  294. static void
  295. unmask_irq (unsigned int irq)
  296. {
  297. unsigned long flags;
  298. char __iomem *addr;
  299. u32 low32;
  300. int rte_index;
  301. ia64_vector vec = irq_to_vector(irq);
  302. struct iosapic_rte_info *rte;
  303. if (list_empty(&iosapic_intr_info[vec].rtes))
  304. return; /* not an IOSAPIC interrupt! */
  305. spin_lock_irqsave(&iosapic_lock, flags);
  306. low32 = iosapic_intr_info[vec].low32 &= ~IOSAPIC_MASK;
  307. list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) {
  308. addr = rte->addr;
  309. rte_index = rte->rte_index;
  310. iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
  311. }
  312. spin_unlock_irqrestore(&iosapic_lock, flags);
  313. }
  314. static void
  315. iosapic_set_affinity (unsigned int irq, cpumask_t mask)
  316. {
  317. #ifdef CONFIG_SMP
  318. unsigned long flags;
  319. u32 high32, low32;
  320. int dest, rte_index;
  321. char __iomem *addr;
  322. int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0;
  323. ia64_vector vec;
  324. struct iosapic_rte_info *rte;
  325. irq &= (~IA64_IRQ_REDIRECTED);
  326. vec = irq_to_vector(irq);
  327. if (cpus_empty(mask))
  328. return;
  329. dest = cpu_physical_id(first_cpu(mask));
  330. if (list_empty(&iosapic_intr_info[vec].rtes))
  331. return; /* not an IOSAPIC interrupt */
  332. set_irq_affinity_info(irq, dest, redir);
  333. /* dest contains both id and eid */
  334. high32 = dest << IOSAPIC_DEST_SHIFT;
  335. spin_lock_irqsave(&iosapic_lock, flags);
  336. low32 = iosapic_intr_info[vec].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
  337. if (redir)
  338. /* change delivery mode to lowest priority */
  339. low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
  340. else
  341. /* change delivery mode to fixed */
  342. low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT);
  343. iosapic_intr_info[vec].low32 = low32;
  344. iosapic_intr_info[vec].dest = dest;
  345. list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) {
  346. addr = rte->addr;
  347. rte_index = rte->rte_index;
  348. iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32);
  349. iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
  350. }
  351. spin_unlock_irqrestore(&iosapic_lock, flags);
  352. #endif
  353. }
  354. /*
  355. * Handlers for level-triggered interrupts.
  356. */
  357. static unsigned int
  358. iosapic_startup_level_irq (unsigned int irq)
  359. {
  360. unmask_irq(irq);
  361. return 0;
  362. }
  363. static void
  364. iosapic_end_level_irq (unsigned int irq)
  365. {
  366. ia64_vector vec = irq_to_vector(irq);
  367. struct iosapic_rte_info *rte;
  368. move_native_irq(irq);
  369. list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list)
  370. iosapic_eoi(rte->addr, vec);
  371. }
  372. #define iosapic_shutdown_level_irq mask_irq
  373. #define iosapic_enable_level_irq unmask_irq
  374. #define iosapic_disable_level_irq mask_irq
  375. #define iosapic_ack_level_irq nop
  376. struct irq_chip irq_type_iosapic_level = {
  377. .name = "IO-SAPIC-level",
  378. .startup = iosapic_startup_level_irq,
  379. .shutdown = iosapic_shutdown_level_irq,
  380. .enable = iosapic_enable_level_irq,
  381. .disable = iosapic_disable_level_irq,
  382. .ack = iosapic_ack_level_irq,
  383. .end = iosapic_end_level_irq,
  384. .mask = mask_irq,
  385. .unmask = unmask_irq,
  386. .set_affinity = iosapic_set_affinity
  387. };
  388. /*
  389. * Handlers for edge-triggered interrupts.
  390. */
  391. static unsigned int
  392. iosapic_startup_edge_irq (unsigned int irq)
  393. {
  394. unmask_irq(irq);
  395. /*
  396. * IOSAPIC simply drops interrupts pended while the
  397. * corresponding pin was masked, so we can't know if an
  398. * interrupt is pending already. Let's hope not...
  399. */
  400. return 0;
  401. }
  402. static void
  403. iosapic_ack_edge_irq (unsigned int irq)
  404. {
  405. irq_desc_t *idesc = irq_desc + irq;
  406. move_native_irq(irq);
  407. /*
  408. * Once we have recorded IRQ_PENDING already, we can mask the
  409. * interrupt for real. This prevents IRQ storms from unhandled
  410. * devices.
  411. */
  412. if ((idesc->status & (IRQ_PENDING|IRQ_DISABLED)) ==
  413. (IRQ_PENDING|IRQ_DISABLED))
  414. mask_irq(irq);
  415. }
  416. #define iosapic_enable_edge_irq unmask_irq
  417. #define iosapic_disable_edge_irq nop
  418. #define iosapic_end_edge_irq nop
  419. struct irq_chip irq_type_iosapic_edge = {
  420. .name = "IO-SAPIC-edge",
  421. .startup = iosapic_startup_edge_irq,
  422. .shutdown = iosapic_disable_edge_irq,
  423. .enable = iosapic_enable_edge_irq,
  424. .disable = iosapic_disable_edge_irq,
  425. .ack = iosapic_ack_edge_irq,
  426. .end = iosapic_end_edge_irq,
  427. .mask = mask_irq,
  428. .unmask = unmask_irq,
  429. .set_affinity = iosapic_set_affinity
  430. };
  431. unsigned int
  432. iosapic_version (char __iomem *addr)
  433. {
  434. /*
  435. * IOSAPIC Version Register return 32 bit structure like:
  436. * {
  437. * unsigned int version : 8;
  438. * unsigned int reserved1 : 8;
  439. * unsigned int max_redir : 8;
  440. * unsigned int reserved2 : 8;
  441. * }
  442. */
  443. return iosapic_read(addr, IOSAPIC_VERSION);
  444. }
  445. static int iosapic_find_sharable_vector (unsigned long trigger,
  446. unsigned long pol)
  447. {
  448. int i, vector = -1, min_count = -1;
  449. struct iosapic_intr_info *info;
  450. /*
  451. * shared vectors for edge-triggered interrupts are not
  452. * supported yet
  453. */
  454. if (trigger == IOSAPIC_EDGE)
  455. return -1;
  456. for (i = IA64_FIRST_DEVICE_VECTOR; i <= IA64_LAST_DEVICE_VECTOR; i++) {
  457. info = &iosapic_intr_info[i];
  458. if (info->trigger == trigger && info->polarity == pol &&
  459. (info->dmode == IOSAPIC_FIXED || info->dmode ==
  460. IOSAPIC_LOWEST_PRIORITY)) {
  461. if (min_count == -1 || info->count < min_count) {
  462. vector = i;
  463. min_count = info->count;
  464. }
  465. }
  466. }
  467. return vector;
  468. }
  469. /*
  470. * if the given vector is already owned by other,
  471. * assign a new vector for the other and make the vector available
  472. */
  473. static void __init
  474. iosapic_reassign_vector (int vector)
  475. {
  476. int new_vector;
  477. if (!list_empty(&iosapic_intr_info[vector].rtes)) {
  478. new_vector = assign_irq_vector(AUTO_ASSIGN);
  479. if (new_vector < 0)
  480. panic("%s: out of interrupt vectors!\n", __FUNCTION__);
  481. printk(KERN_INFO "Reassigning vector %d to %d\n",
  482. vector, new_vector);
  483. memcpy(&iosapic_intr_info[new_vector], &iosapic_intr_info[vector],
  484. sizeof(struct iosapic_intr_info));
  485. INIT_LIST_HEAD(&iosapic_intr_info[new_vector].rtes);
  486. list_move(iosapic_intr_info[vector].rtes.next,
  487. &iosapic_intr_info[new_vector].rtes);
  488. memset(&iosapic_intr_info[vector], 0,
  489. sizeof(struct iosapic_intr_info));
  490. iosapic_intr_info[vector].low32 = IOSAPIC_MASK;
  491. INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes);
  492. }
  493. }
  494. static struct iosapic_rte_info *iosapic_alloc_rte (void)
  495. {
  496. int i;
  497. struct iosapic_rte_info *rte;
  498. int preallocated = 0;
  499. if (!iosapic_kmalloc_ok && list_empty(&free_rte_list)) {
  500. rte = alloc_bootmem(sizeof(struct iosapic_rte_info) *
  501. NR_PREALLOCATE_RTE_ENTRIES);
  502. if (!rte)
  503. return NULL;
  504. for (i = 0; i < NR_PREALLOCATE_RTE_ENTRIES; i++, rte++)
  505. list_add(&rte->rte_list, &free_rte_list);
  506. }
  507. if (!list_empty(&free_rte_list)) {
  508. rte = list_entry(free_rte_list.next, struct iosapic_rte_info,
  509. rte_list);
  510. list_del(&rte->rte_list);
  511. preallocated++;
  512. } else {
  513. rte = kmalloc(sizeof(struct iosapic_rte_info), GFP_ATOMIC);
  514. if (!rte)
  515. return NULL;
  516. }
  517. memset(rte, 0, sizeof(struct iosapic_rte_info));
  518. if (preallocated)
  519. rte->flags |= RTE_PREALLOCATED;
  520. return rte;
  521. }
  522. static void iosapic_free_rte (struct iosapic_rte_info *rte)
  523. {
  524. if (rte->flags & RTE_PREALLOCATED)
  525. list_add_tail(&rte->rte_list, &free_rte_list);
  526. else
  527. kfree(rte);
  528. }
  529. static inline int vector_is_shared (int vector)
  530. {
  531. return (iosapic_intr_info[vector].count > 1);
  532. }
  533. static int
  534. register_intr (unsigned int gsi, int vector, unsigned char delivery,
  535. unsigned long polarity, unsigned long trigger)
  536. {
  537. irq_desc_t *idesc;
  538. struct hw_interrupt_type *irq_type;
  539. int rte_index;
  540. int index;
  541. unsigned long gsi_base;
  542. void __iomem *iosapic_address;
  543. struct iosapic_rte_info *rte;
  544. index = find_iosapic(gsi);
  545. if (index < 0) {
  546. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  547. __FUNCTION__, gsi);
  548. return -ENODEV;
  549. }
  550. iosapic_address = iosapic_lists[index].addr;
  551. gsi_base = iosapic_lists[index].gsi_base;
  552. rte = gsi_vector_to_rte(gsi, vector);
  553. if (!rte) {
  554. rte = iosapic_alloc_rte();
  555. if (!rte) {
  556. printk(KERN_WARNING "%s: cannot allocate memory\n",
  557. __FUNCTION__);
  558. return -ENOMEM;
  559. }
  560. rte_index = gsi - gsi_base;
  561. rte->rte_index = rte_index;
  562. rte->addr = iosapic_address;
  563. rte->gsi_base = gsi_base;
  564. rte->refcnt++;
  565. list_add_tail(&rte->rte_list, &iosapic_intr_info[vector].rtes);
  566. iosapic_intr_info[vector].count++;
  567. iosapic_lists[index].rtes_inuse++;
  568. }
  569. else if (vector_is_shared(vector)) {
  570. struct iosapic_intr_info *info = &iosapic_intr_info[vector];
  571. if (info->trigger != trigger || info->polarity != polarity) {
  572. printk (KERN_WARNING
  573. "%s: cannot override the interrupt\n",
  574. __FUNCTION__);
  575. return -EINVAL;
  576. }
  577. }
  578. iosapic_intr_info[vector].polarity = polarity;
  579. iosapic_intr_info[vector].dmode = delivery;
  580. iosapic_intr_info[vector].trigger = trigger;
  581. if (trigger == IOSAPIC_EDGE)
  582. irq_type = &irq_type_iosapic_edge;
  583. else
  584. irq_type = &irq_type_iosapic_level;
  585. idesc = irq_desc + vector;
  586. if (idesc->chip != irq_type) {
  587. if (idesc->chip != &no_irq_type)
  588. printk(KERN_WARNING
  589. "%s: changing vector %d from %s to %s\n",
  590. __FUNCTION__, vector,
  591. idesc->chip->name, irq_type->name);
  592. idesc->chip = irq_type;
  593. }
  594. return 0;
  595. }
  596. static unsigned int
  597. get_target_cpu (unsigned int gsi, int vector)
  598. {
  599. #ifdef CONFIG_SMP
  600. static int cpu = -1;
  601. extern int cpe_vector;
  602. /*
  603. * In case of vector shared by multiple RTEs, all RTEs that
  604. * share the vector need to use the same destination CPU.
  605. */
  606. if (!list_empty(&iosapic_intr_info[vector].rtes))
  607. return iosapic_intr_info[vector].dest;
  608. /*
  609. * If the platform supports redirection via XTP, let it
  610. * distribute interrupts.
  611. */
  612. if (smp_int_redirect & SMP_IRQ_REDIRECTION)
  613. return cpu_physical_id(smp_processor_id());
  614. /*
  615. * Some interrupts (ACPI SCI, for instance) are registered
  616. * before the BSP is marked as online.
  617. */
  618. if (!cpu_online(smp_processor_id()))
  619. return cpu_physical_id(smp_processor_id());
  620. #ifdef CONFIG_ACPI
  621. if (cpe_vector > 0 && vector == IA64_CPEP_VECTOR)
  622. return get_cpei_target_cpu();
  623. #endif
  624. #ifdef CONFIG_NUMA
  625. {
  626. int num_cpus, cpu_index, iosapic_index, numa_cpu, i = 0;
  627. cpumask_t cpu_mask;
  628. iosapic_index = find_iosapic(gsi);
  629. if (iosapic_index < 0 ||
  630. iosapic_lists[iosapic_index].node == MAX_NUMNODES)
  631. goto skip_numa_setup;
  632. cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node);
  633. for_each_cpu_mask(numa_cpu, cpu_mask) {
  634. if (!cpu_online(numa_cpu))
  635. cpu_clear(numa_cpu, cpu_mask);
  636. }
  637. num_cpus = cpus_weight(cpu_mask);
  638. if (!num_cpus)
  639. goto skip_numa_setup;
  640. /* Use vector assignment to distribute across cpus in node */
  641. cpu_index = vector % num_cpus;
  642. for (numa_cpu = first_cpu(cpu_mask) ; i < cpu_index ; i++)
  643. numa_cpu = next_cpu(numa_cpu, cpu_mask);
  644. if (numa_cpu != NR_CPUS)
  645. return cpu_physical_id(numa_cpu);
  646. }
  647. skip_numa_setup:
  648. #endif
  649. /*
  650. * Otherwise, round-robin interrupt vectors across all the
  651. * processors. (It'd be nice if we could be smarter in the
  652. * case of NUMA.)
  653. */
  654. do {
  655. if (++cpu >= NR_CPUS)
  656. cpu = 0;
  657. } while (!cpu_online(cpu));
  658. return cpu_physical_id(cpu);
  659. #else /* CONFIG_SMP */
  660. return cpu_physical_id(smp_processor_id());
  661. #endif
  662. }
  663. /*
  664. * ACPI can describe IOSAPIC interrupts via static tables and namespace
  665. * methods. This provides an interface to register those interrupts and
  666. * program the IOSAPIC RTE.
  667. */
  668. int
  669. iosapic_register_intr (unsigned int gsi,
  670. unsigned long polarity, unsigned long trigger)
  671. {
  672. int vector, mask = 1, err;
  673. unsigned int dest;
  674. unsigned long flags;
  675. struct iosapic_rte_info *rte;
  676. u32 low32;
  677. again:
  678. /*
  679. * If this GSI has already been registered (i.e., it's a
  680. * shared interrupt, or we lost a race to register it),
  681. * don't touch the RTE.
  682. */
  683. spin_lock_irqsave(&iosapic_lock, flags);
  684. vector = gsi_to_vector(gsi);
  685. if (vector > 0) {
  686. rte = gsi_vector_to_rte(gsi, vector);
  687. rte->refcnt++;
  688. spin_unlock_irqrestore(&iosapic_lock, flags);
  689. return vector;
  690. }
  691. spin_unlock_irqrestore(&iosapic_lock, flags);
  692. /* If vector is running out, we try to find a sharable vector */
  693. vector = assign_irq_vector(AUTO_ASSIGN);
  694. if (vector < 0) {
  695. vector = iosapic_find_sharable_vector(trigger, polarity);
  696. if (vector < 0)
  697. return -ENOSPC;
  698. }
  699. spin_lock_irqsave(&irq_desc[vector].lock, flags);
  700. spin_lock(&iosapic_lock);
  701. if (gsi_to_vector(gsi) > 0) {
  702. if (list_empty(&iosapic_intr_info[vector].rtes))
  703. free_irq_vector(vector);
  704. spin_unlock(&iosapic_lock);
  705. spin_unlock_irqrestore(&irq_desc[vector].lock, flags);
  706. goto again;
  707. }
  708. dest = get_target_cpu(gsi, vector);
  709. err = register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY,
  710. polarity, trigger);
  711. if (err < 0) {
  712. spin_unlock(&iosapic_lock);
  713. spin_unlock_irqrestore(&irq_desc[vector].lock, flags);
  714. return err;
  715. }
  716. /*
  717. * If the vector is shared and already unmasked for other
  718. * interrupt sources, don't mask it.
  719. */
  720. low32 = iosapic_intr_info[vector].low32;
  721. if (vector_is_shared(vector) && !(low32 & IOSAPIC_MASK))
  722. mask = 0;
  723. set_rte(gsi, vector, dest, mask);
  724. spin_unlock(&iosapic_lock);
  725. spin_unlock_irqrestore(&irq_desc[vector].lock, flags);
  726. printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n",
  727. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  728. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  729. cpu_logical_id(dest), dest, vector);
  730. return vector;
  731. }
  732. void
  733. iosapic_unregister_intr (unsigned int gsi)
  734. {
  735. unsigned long flags;
  736. int irq, vector, index;
  737. irq_desc_t *idesc;
  738. u32 low32;
  739. unsigned long trigger, polarity;
  740. unsigned int dest;
  741. struct iosapic_rte_info *rte;
  742. /*
  743. * If the irq associated with the gsi is not found,
  744. * iosapic_unregister_intr() is unbalanced. We need to check
  745. * this again after getting locks.
  746. */
  747. irq = gsi_to_irq(gsi);
  748. if (irq < 0) {
  749. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  750. gsi);
  751. WARN_ON(1);
  752. return;
  753. }
  754. vector = irq_to_vector(irq);
  755. idesc = irq_desc + irq;
  756. spin_lock_irqsave(&idesc->lock, flags);
  757. spin_lock(&iosapic_lock);
  758. if ((rte = gsi_vector_to_rte(gsi, vector)) == NULL) {
  759. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  760. gsi);
  761. WARN_ON(1);
  762. goto out;
  763. }
  764. if (--rte->refcnt > 0)
  765. goto out;
  766. /* Mask the interrupt */
  767. low32 = iosapic_intr_info[vector].low32 | IOSAPIC_MASK;
  768. iosapic_write(rte->addr, IOSAPIC_RTE_LOW(rte->rte_index), low32);
  769. /* Remove the rte entry from the list */
  770. list_del(&rte->rte_list);
  771. iosapic_intr_info[vector].count--;
  772. iosapic_free_rte(rte);
  773. index = find_iosapic(gsi);
  774. iosapic_lists[index].rtes_inuse--;
  775. WARN_ON(iosapic_lists[index].rtes_inuse < 0);
  776. trigger = iosapic_intr_info[vector].trigger;
  777. polarity = iosapic_intr_info[vector].polarity;
  778. dest = iosapic_intr_info[vector].dest;
  779. printk(KERN_INFO
  780. "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n",
  781. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  782. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  783. cpu_logical_id(dest), dest, vector);
  784. if (list_empty(&iosapic_intr_info[vector].rtes)) {
  785. /* Sanity check */
  786. BUG_ON(iosapic_intr_info[vector].count);
  787. /* Clear the interrupt controller descriptor */
  788. idesc->chip = &no_irq_type;
  789. #ifdef CONFIG_SMP
  790. /* Clear affinity */
  791. cpus_setall(idesc->affinity);
  792. #endif
  793. /* Clear the interrupt information */
  794. memset(&iosapic_intr_info[vector], 0,
  795. sizeof(struct iosapic_intr_info));
  796. iosapic_intr_info[vector].low32 |= IOSAPIC_MASK;
  797. INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes);
  798. if (idesc->action) {
  799. printk(KERN_ERR
  800. "interrupt handlers still exist on IRQ %u\n",
  801. irq);
  802. WARN_ON(1);
  803. }
  804. /* Free the interrupt vector */
  805. free_irq_vector(vector);
  806. }
  807. out:
  808. spin_unlock(&iosapic_lock);
  809. spin_unlock_irqrestore(&idesc->lock, flags);
  810. }
  811. /*
  812. * ACPI calls this when it finds an entry for a platform interrupt.
  813. */
  814. int __init
  815. iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
  816. int iosapic_vector, u16 eid, u16 id,
  817. unsigned long polarity, unsigned long trigger)
  818. {
  819. static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"};
  820. unsigned char delivery;
  821. int vector, mask = 0;
  822. unsigned int dest = ((id << 8) | eid) & 0xffff;
  823. switch (int_type) {
  824. case ACPI_INTERRUPT_PMI:
  825. vector = iosapic_vector;
  826. /*
  827. * since PMI vector is alloc'd by FW(ACPI) not by kernel,
  828. * we need to make sure the vector is available
  829. */
  830. iosapic_reassign_vector(vector);
  831. delivery = IOSAPIC_PMI;
  832. break;
  833. case ACPI_INTERRUPT_INIT:
  834. vector = assign_irq_vector(AUTO_ASSIGN);
  835. if (vector < 0)
  836. panic("%s: out of interrupt vectors!\n", __FUNCTION__);
  837. delivery = IOSAPIC_INIT;
  838. break;
  839. case ACPI_INTERRUPT_CPEI:
  840. vector = IA64_CPE_VECTOR;
  841. delivery = IOSAPIC_LOWEST_PRIORITY;
  842. mask = 1;
  843. break;
  844. default:
  845. printk(KERN_ERR "%s: invalid int type 0x%x\n", __FUNCTION__,
  846. int_type);
  847. return -1;
  848. }
  849. register_intr(gsi, vector, delivery, polarity, trigger);
  850. printk(KERN_INFO
  851. "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)"
  852. " vector %d\n",
  853. int_type < ARRAY_SIZE(name) ? name[int_type] : "unknown",
  854. int_type, gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  855. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  856. cpu_logical_id(dest), dest, vector);
  857. set_rte(gsi, vector, dest, mask);
  858. return vector;
  859. }
  860. /*
  861. * ACPI calls this when it finds an entry for a legacy ISA IRQ override.
  862. */
  863. void __devinit
  864. iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi,
  865. unsigned long polarity,
  866. unsigned long trigger)
  867. {
  868. int vector;
  869. unsigned int dest = cpu_physical_id(smp_processor_id());
  870. vector = isa_irq_to_vector(isa_irq);
  871. register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, polarity, trigger);
  872. DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n",
  873. isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level",
  874. polarity == IOSAPIC_POL_HIGH ? "high" : "low",
  875. cpu_logical_id(dest), dest, vector);
  876. set_rte(gsi, vector, dest, 1);
  877. }
  878. void __init
  879. iosapic_system_init (int system_pcat_compat)
  880. {
  881. int vector;
  882. for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) {
  883. iosapic_intr_info[vector].low32 = IOSAPIC_MASK;
  884. /* mark as unused */
  885. INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes);
  886. }
  887. pcat_compat = system_pcat_compat;
  888. if (pcat_compat) {
  889. /*
  890. * Disable the compatibility mode interrupts (8259 style),
  891. * needs IN/OUT support enabled.
  892. */
  893. printk(KERN_INFO
  894. "%s: Disabling PC-AT compatible 8259 interrupts\n",
  895. __FUNCTION__);
  896. outb(0xff, 0xA1);
  897. outb(0xff, 0x21);
  898. }
  899. }
  900. static inline int
  901. iosapic_alloc (void)
  902. {
  903. int index;
  904. for (index = 0; index < NR_IOSAPICS; index++)
  905. if (!iosapic_lists[index].addr)
  906. return index;
  907. printk(KERN_WARNING "%s: failed to allocate iosapic\n", __FUNCTION__);
  908. return -1;
  909. }
  910. static inline void
  911. iosapic_free (int index)
  912. {
  913. memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0]));
  914. }
  915. static inline int
  916. iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver)
  917. {
  918. int index;
  919. unsigned int gsi_end, base, end;
  920. /* check gsi range */
  921. gsi_end = gsi_base + ((ver >> 16) & 0xff);
  922. for (index = 0; index < NR_IOSAPICS; index++) {
  923. if (!iosapic_lists[index].addr)
  924. continue;
  925. base = iosapic_lists[index].gsi_base;
  926. end = base + iosapic_lists[index].num_rte - 1;
  927. if (gsi_end < base || end < gsi_base)
  928. continue; /* OK */
  929. return -EBUSY;
  930. }
  931. return 0;
  932. }
  933. int __devinit
  934. iosapic_init (unsigned long phys_addr, unsigned int gsi_base)
  935. {
  936. int num_rte, err, index;
  937. unsigned int isa_irq, ver;
  938. char __iomem *addr;
  939. unsigned long flags;
  940. spin_lock_irqsave(&iosapic_lock, flags);
  941. addr = ioremap(phys_addr, 0);
  942. ver = iosapic_version(addr);
  943. if ((err = iosapic_check_gsi_range(gsi_base, ver))) {
  944. iounmap(addr);
  945. spin_unlock_irqrestore(&iosapic_lock, flags);
  946. return err;
  947. }
  948. /*
  949. * The MAX_REDIR register holds the highest input pin number
  950. * (starting from 0). We add 1 so that we can use it for
  951. * number of pins (= RTEs)
  952. */
  953. num_rte = ((ver >> 16) & 0xff) + 1;
  954. index = iosapic_alloc();
  955. iosapic_lists[index].addr = addr;
  956. iosapic_lists[index].gsi_base = gsi_base;
  957. iosapic_lists[index].num_rte = num_rte;
  958. #ifdef CONFIG_NUMA
  959. iosapic_lists[index].node = MAX_NUMNODES;
  960. #endif
  961. spin_unlock_irqrestore(&iosapic_lock, flags);
  962. if ((gsi_base == 0) && pcat_compat) {
  963. /*
  964. * Map the legacy ISA devices into the IOSAPIC data. Some of
  965. * these may get reprogrammed later on with data from the ACPI
  966. * Interrupt Source Override table.
  967. */
  968. for (isa_irq = 0; isa_irq < 16; ++isa_irq)
  969. iosapic_override_isa_irq(isa_irq, isa_irq,
  970. IOSAPIC_POL_HIGH,
  971. IOSAPIC_EDGE);
  972. }
  973. return 0;
  974. }
  975. #ifdef CONFIG_HOTPLUG
  976. int
  977. iosapic_remove (unsigned int gsi_base)
  978. {
  979. int index, err = 0;
  980. unsigned long flags;
  981. spin_lock_irqsave(&iosapic_lock, flags);
  982. index = find_iosapic(gsi_base);
  983. if (index < 0) {
  984. printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n",
  985. __FUNCTION__, gsi_base);
  986. goto out;
  987. }
  988. if (iosapic_lists[index].rtes_inuse) {
  989. err = -EBUSY;
  990. printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n",
  991. __FUNCTION__, gsi_base);
  992. goto out;
  993. }
  994. iounmap(iosapic_lists[index].addr);
  995. iosapic_free(index);
  996. out:
  997. spin_unlock_irqrestore(&iosapic_lock, flags);
  998. return err;
  999. }
  1000. #endif /* CONFIG_HOTPLUG */
  1001. #ifdef CONFIG_NUMA
  1002. void __devinit
  1003. map_iosapic_to_node(unsigned int gsi_base, int node)
  1004. {
  1005. int index;
  1006. index = find_iosapic(gsi_base);
  1007. if (index < 0) {
  1008. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  1009. __FUNCTION__, gsi_base);
  1010. return;
  1011. }
  1012. iosapic_lists[index].node = node;
  1013. return;
  1014. }
  1015. #endif
  1016. static int __init iosapic_enable_kmalloc (void)
  1017. {
  1018. iosapic_kmalloc_ok = 1;
  1019. return 0;
  1020. }
  1021. core_initcall (iosapic_enable_kmalloc);