iosapic.c 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121
  1. /*
  2. * I/O SAPIC support.
  3. *
  4. * Copyright (C) 1999 Intel Corp.
  5. * Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com>
  6. * Copyright (C) 2000-2002 J.I. Lee <jung-ik.lee@intel.com>
  7. * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co.
  8. * David Mosberger-Tang <davidm@hpl.hp.com>
  9. * Copyright (C) 1999 VA Linux Systems
  10. * Copyright (C) 1999,2000 Walt Drummond <drummond@valinux.com>
  11. *
  12. * 00/04/19 D. Mosberger Rewritten to mirror more closely the x86 I/O
  13. * APIC code. In particular, we now have separate
  14. * handlers for edge and level triggered
  15. * interrupts.
  16. * 00/10/27 Asit Mallick, Goutham Rao <goutham.rao@intel.com> IRQ vector
  17. * allocation PCI to vector mapping, shared PCI
  18. * interrupts.
  19. * 00/10/27 D. Mosberger Document things a bit more to make them more
  20. * understandable. Clean up much of the old
  21. * IOSAPIC cruft.
  22. * 01/07/27 J.I. Lee PCI irq routing, Platform/Legacy interrupts
  23. * and fixes for ACPI S5(SoftOff) support.
  24. * 02/01/23 J.I. Lee iosapic pgm fixes for PCI irq routing from _PRT
  25. * 02/01/07 E. Focht <efocht@ess.nec.de> Redirectable interrupt
  26. * vectors in iosapic_set_affinity(),
  27. * initializations for /proc/irq/#/smp_affinity
  28. * 02/04/02 P. Diefenbaugh Cleaned up ACPI PCI IRQ routing.
  29. * 02/04/18 J.I. Lee bug fix in iosapic_init_pci_irq
  30. * 02/04/30 J.I. Lee bug fix in find_iosapic to fix ACPI PCI IRQ to
  31. * IOSAPIC mapping error
  32. * 02/07/29 T. Kochi Allocate interrupt vectors dynamically
  33. * 02/08/04 T. Kochi Cleaned up terminology (irq, global system
  34. * interrupt, vector, etc.)
  35. * 02/09/20 D. Mosberger Simplified by taking advantage of ACPI's
  36. * pci_irq code.
  37. * 03/02/19 B. Helgaas Make pcat_compat system-wide, not per-IOSAPIC.
  38. * Remove iosapic_address & gsi_base from
  39. * external interfaces. Rationalize
  40. * __init/__devinit attributes.
  41. * 04/12/04 Ashok Raj <ashok.raj@intel.com> Intel Corporation 2004
  42. * Updated to work with irq migration necessary
  43. * for CPU Hotplug
  44. */
  45. /*
  46. * Here is what the interrupt logic between a PCI device and the kernel looks
  47. * like:
  48. *
  49. * (1) A PCI device raises one of the four interrupt pins (INTA, INTB, INTC,
  50. * INTD). The device is uniquely identified by its bus-, and slot-number
  51. * (the function number does not matter here because all functions share
  52. * the same interrupt lines).
  53. *
  54. * (2) The motherboard routes the interrupt line to a pin on a IOSAPIC
  55. * controller. Multiple interrupt lines may have to share the same
  56. * IOSAPIC pin (if they're level triggered and use the same polarity).
  57. * Each interrupt line has a unique Global System Interrupt (GSI) number
  58. * which can be calculated as the sum of the controller's base GSI number
  59. * and the IOSAPIC pin number to which the line connects.
  60. *
  61. * (3) The IOSAPIC uses an internal routing table entries (RTEs) to map the
  62. * IOSAPIC pin into the IA-64 interrupt vector. This interrupt vector is then
  63. * sent to the CPU.
  64. *
  65. * (4) The kernel recognizes an interrupt as an IRQ. The IRQ interface is
  66. * used as architecture-independent interrupt handling mechanism in Linux.
  67. * As an IRQ is a number, we have to have
  68. * IA-64 interrupt vector number <-> IRQ number mapping. On smaller
  69. * systems, we use one-to-one mapping between IA-64 vector and IRQ. A
  70. * platform can implement platform_irq_to_vector(irq) and
  71. * platform_local_vector_to_irq(vector) APIs to differentiate the mapping.
  72. * Please see also arch/ia64/include/asm/hw_irq.h for those APIs.
  73. *
  74. * To sum up, there are three levels of mappings involved:
  75. *
  76. * PCI pin -> global system interrupt (GSI) -> IA-64 vector <-> IRQ
  77. *
  78. * Note: The term "IRQ" is loosely used everywhere in Linux kernel to
  79. * describeinterrupts. Now we use "IRQ" only for Linux IRQ's. ISA IRQ
  80. * (isa_irq) is the only exception in this source code.
  81. */
  82. #include <linux/acpi.h>
  83. #include <linux/init.h>
  84. #include <linux/irq.h>
  85. #include <linux/kernel.h>
  86. #include <linux/list.h>
  87. #include <linux/pci.h>
  88. #include <linux/slab.h>
  89. #include <linux/smp.h>
  90. #include <linux/string.h>
  91. #include <linux/bootmem.h>
  92. #include <asm/delay.h>
  93. #include <asm/hw_irq.h>
  94. #include <asm/io.h>
  95. #include <asm/iosapic.h>
  96. #include <asm/machvec.h>
  97. #include <asm/processor.h>
  98. #include <asm/ptrace.h>
  99. #include <asm/system.h>
  100. #undef DEBUG_INTERRUPT_ROUTING
  101. #ifdef DEBUG_INTERRUPT_ROUTING
  102. #define DBG(fmt...) printk(fmt)
  103. #else
  104. #define DBG(fmt...)
  105. #endif
  106. static DEFINE_SPINLOCK(iosapic_lock);
  107. /*
  108. * These tables map IA-64 vectors to the IOSAPIC pin that generates this
  109. * vector.
  110. */
  111. #define NO_REF_RTE 0
  112. static struct iosapic {
  113. char __iomem *addr; /* base address of IOSAPIC */
  114. unsigned int gsi_base; /* GSI base */
  115. unsigned short num_rte; /* # of RTEs on this IOSAPIC */
  116. int rtes_inuse; /* # of RTEs in use on this IOSAPIC */
  117. #ifdef CONFIG_NUMA
  118. unsigned short node; /* numa node association via pxm */
  119. #endif
  120. spinlock_t lock; /* lock for indirect reg access */
  121. } iosapic_lists[NR_IOSAPICS];
  122. struct iosapic_rte_info {
  123. struct list_head rte_list; /* RTEs sharing the same vector */
  124. char rte_index; /* IOSAPIC RTE index */
  125. int refcnt; /* reference counter */
  126. struct iosapic *iosapic;
  127. } ____cacheline_aligned;
  128. static struct iosapic_intr_info {
  129. struct list_head rtes; /* RTEs using this vector (empty =>
  130. * not an IOSAPIC interrupt) */
  131. int count; /* # of registered RTEs */
  132. u32 low32; /* current value of low word of
  133. * Redirection table entry */
  134. unsigned int dest; /* destination CPU physical ID */
  135. unsigned char dmode : 3; /* delivery mode (see iosapic.h) */
  136. unsigned char polarity: 1; /* interrupt polarity
  137. * (see iosapic.h) */
  138. unsigned char trigger : 1; /* trigger mode (see iosapic.h) */
  139. } iosapic_intr_info[NR_IRQS];
  140. static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */
  141. static inline void
  142. iosapic_write(struct iosapic *iosapic, unsigned int reg, u32 val)
  143. {
  144. unsigned long flags;
  145. spin_lock_irqsave(&iosapic->lock, flags);
  146. __iosapic_write(iosapic->addr, reg, val);
  147. spin_unlock_irqrestore(&iosapic->lock, flags);
  148. }
  149. /*
  150. * Find an IOSAPIC associated with a GSI
  151. */
  152. static inline int
  153. find_iosapic (unsigned int gsi)
  154. {
  155. int i;
  156. for (i = 0; i < NR_IOSAPICS; i++) {
  157. if ((unsigned) (gsi - iosapic_lists[i].gsi_base) <
  158. iosapic_lists[i].num_rte)
  159. return i;
  160. }
  161. return -1;
  162. }
  163. static inline int __gsi_to_irq(unsigned int gsi)
  164. {
  165. int irq;
  166. struct iosapic_intr_info *info;
  167. struct iosapic_rte_info *rte;
  168. for (irq = 0; irq < NR_IRQS; irq++) {
  169. info = &iosapic_intr_info[irq];
  170. list_for_each_entry(rte, &info->rtes, rte_list)
  171. if (rte->iosapic->gsi_base + rte->rte_index == gsi)
  172. return irq;
  173. }
  174. return -1;
  175. }
  176. int
  177. gsi_to_irq (unsigned int gsi)
  178. {
  179. unsigned long flags;
  180. int irq;
  181. spin_lock_irqsave(&iosapic_lock, flags);
  182. irq = __gsi_to_irq(gsi);
  183. spin_unlock_irqrestore(&iosapic_lock, flags);
  184. return irq;
  185. }
  186. static struct iosapic_rte_info *find_rte(unsigned int irq, unsigned int gsi)
  187. {
  188. struct iosapic_rte_info *rte;
  189. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
  190. if (rte->iosapic->gsi_base + rte->rte_index == gsi)
  191. return rte;
  192. return NULL;
  193. }
  194. static void
  195. set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask)
  196. {
  197. unsigned long pol, trigger, dmode;
  198. u32 low32, high32;
  199. int rte_index;
  200. char redir;
  201. struct iosapic_rte_info *rte;
  202. ia64_vector vector = irq_to_vector(irq);
  203. DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest);
  204. rte = find_rte(irq, gsi);
  205. if (!rte)
  206. return; /* not an IOSAPIC interrupt */
  207. rte_index = rte->rte_index;
  208. pol = iosapic_intr_info[irq].polarity;
  209. trigger = iosapic_intr_info[irq].trigger;
  210. dmode = iosapic_intr_info[irq].dmode;
  211. redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
  212. #ifdef CONFIG_SMP
  213. set_irq_affinity_info(irq, (int)(dest & 0xffff), redir);
  214. #endif
  215. low32 = ((pol << IOSAPIC_POLARITY_SHIFT) |
  216. (trigger << IOSAPIC_TRIGGER_SHIFT) |
  217. (dmode << IOSAPIC_DELIVERY_SHIFT) |
  218. ((mask ? 1 : 0) << IOSAPIC_MASK_SHIFT) |
  219. vector);
  220. /* dest contains both id and eid */
  221. high32 = (dest << IOSAPIC_DEST_SHIFT);
  222. iosapic_write(rte->iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
  223. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  224. iosapic_intr_info[irq].low32 = low32;
  225. iosapic_intr_info[irq].dest = dest;
  226. }
  227. static void
  228. nop (struct irq_data *data)
  229. {
  230. /* do nothing... */
  231. }
  232. #ifdef CONFIG_KEXEC
  233. void
  234. kexec_disable_iosapic(void)
  235. {
  236. struct iosapic_intr_info *info;
  237. struct iosapic_rte_info *rte;
  238. ia64_vector vec;
  239. int irq;
  240. for (irq = 0; irq < NR_IRQS; irq++) {
  241. info = &iosapic_intr_info[irq];
  242. vec = irq_to_vector(irq);
  243. list_for_each_entry(rte, &info->rtes,
  244. rte_list) {
  245. iosapic_write(rte->iosapic,
  246. IOSAPIC_RTE_LOW(rte->rte_index),
  247. IOSAPIC_MASK|vec);
  248. iosapic_eoi(rte->iosapic->addr, vec);
  249. }
  250. }
  251. }
  252. #endif
  253. static void
  254. mask_irq (struct irq_data *data)
  255. {
  256. unsigned int irq = data->irq;
  257. u32 low32;
  258. int rte_index;
  259. struct iosapic_rte_info *rte;
  260. if (!iosapic_intr_info[irq].count)
  261. return; /* not an IOSAPIC interrupt! */
  262. /* set only the mask bit */
  263. low32 = iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
  264. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  265. rte_index = rte->rte_index;
  266. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  267. }
  268. }
  269. static void
  270. unmask_irq (struct irq_data *data)
  271. {
  272. unsigned int irq = data->irq;
  273. u32 low32;
  274. int rte_index;
  275. struct iosapic_rte_info *rte;
  276. if (!iosapic_intr_info[irq].count)
  277. return; /* not an IOSAPIC interrupt! */
  278. low32 = iosapic_intr_info[irq].low32 &= ~IOSAPIC_MASK;
  279. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  280. rte_index = rte->rte_index;
  281. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  282. }
  283. }
  284. static int
  285. iosapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
  286. bool force)
  287. {
  288. #ifdef CONFIG_SMP
  289. unsigned int irq = data->irq;
  290. u32 high32, low32;
  291. int cpu, dest, rte_index;
  292. int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0;
  293. struct iosapic_rte_info *rte;
  294. struct iosapic *iosapic;
  295. irq &= (~IA64_IRQ_REDIRECTED);
  296. cpu = cpumask_first_and(cpu_online_mask, mask);
  297. if (cpu >= nr_cpu_ids)
  298. return -1;
  299. if (irq_prepare_move(irq, cpu))
  300. return -1;
  301. dest = cpu_physical_id(cpu);
  302. if (!iosapic_intr_info[irq].count)
  303. return -1; /* not an IOSAPIC interrupt */
  304. set_irq_affinity_info(irq, dest, redir);
  305. /* dest contains both id and eid */
  306. high32 = dest << IOSAPIC_DEST_SHIFT;
  307. low32 = iosapic_intr_info[irq].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
  308. if (redir)
  309. /* change delivery mode to lowest priority */
  310. low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
  311. else
  312. /* change delivery mode to fixed */
  313. low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT);
  314. low32 &= IOSAPIC_VECTOR_MASK;
  315. low32 |= irq_to_vector(irq);
  316. iosapic_intr_info[irq].low32 = low32;
  317. iosapic_intr_info[irq].dest = dest;
  318. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  319. iosapic = rte->iosapic;
  320. rte_index = rte->rte_index;
  321. iosapic_write(iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
  322. iosapic_write(iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  323. }
  324. #endif
  325. return 0;
  326. }
  327. /*
  328. * Handlers for level-triggered interrupts.
  329. */
  330. static unsigned int
  331. iosapic_startup_level_irq (struct irq_data *data)
  332. {
  333. unmask_irq(data);
  334. return 0;
  335. }
  336. static void
  337. iosapic_unmask_level_irq (struct irq_data *data)
  338. {
  339. unsigned int irq = data->irq;
  340. ia64_vector vec = irq_to_vector(irq);
  341. struct iosapic_rte_info *rte;
  342. int do_unmask_irq = 0;
  343. irq_complete_move(irq);
  344. if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) {
  345. do_unmask_irq = 1;
  346. mask_irq(data);
  347. } else
  348. unmask_irq(data);
  349. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
  350. iosapic_eoi(rte->iosapic->addr, vec);
  351. if (unlikely(do_unmask_irq)) {
  352. move_masked_irq(irq);
  353. unmask_irq(data);
  354. }
  355. }
  356. #define iosapic_shutdown_level_irq mask_irq
  357. #define iosapic_enable_level_irq unmask_irq
  358. #define iosapic_disable_level_irq mask_irq
  359. #define iosapic_ack_level_irq nop
  360. static struct irq_chip irq_type_iosapic_level = {
  361. .name = "IO-SAPIC-level",
  362. .irq_startup = iosapic_startup_level_irq,
  363. .irq_shutdown = iosapic_shutdown_level_irq,
  364. .irq_enable = iosapic_enable_level_irq,
  365. .irq_disable = iosapic_disable_level_irq,
  366. .irq_ack = iosapic_ack_level_irq,
  367. .irq_mask = mask_irq,
  368. .irq_unmask = iosapic_unmask_level_irq,
  369. .irq_set_affinity = iosapic_set_affinity
  370. };
  371. /*
  372. * Handlers for edge-triggered interrupts.
  373. */
  374. static unsigned int
  375. iosapic_startup_edge_irq (struct irq_data *data)
  376. {
  377. unmask_irq(data);
  378. /*
  379. * IOSAPIC simply drops interrupts pended while the
  380. * corresponding pin was masked, so we can't know if an
  381. * interrupt is pending already. Let's hope not...
  382. */
  383. return 0;
  384. }
  385. static void
  386. iosapic_ack_edge_irq (struct irq_data *data)
  387. {
  388. unsigned int irq = data->irq;
  389. irq_complete_move(irq);
  390. move_native_irq(irq);
  391. }
  392. #define iosapic_enable_edge_irq unmask_irq
  393. #define iosapic_disable_edge_irq nop
  394. static struct irq_chip irq_type_iosapic_edge = {
  395. .name = "IO-SAPIC-edge",
  396. .irq_startup = iosapic_startup_edge_irq,
  397. .irq_shutdown = iosapic_disable_edge_irq,
  398. .irq_enable = iosapic_enable_edge_irq,
  399. .irq_disable = iosapic_disable_edge_irq,
  400. .irq_ack = iosapic_ack_edge_irq,
  401. .irq_mask = mask_irq,
  402. .irq_unmask = unmask_irq,
  403. .irq_set_affinity = iosapic_set_affinity
  404. };
  405. static unsigned int
  406. iosapic_version (char __iomem *addr)
  407. {
  408. /*
  409. * IOSAPIC Version Register return 32 bit structure like:
  410. * {
  411. * unsigned int version : 8;
  412. * unsigned int reserved1 : 8;
  413. * unsigned int max_redir : 8;
  414. * unsigned int reserved2 : 8;
  415. * }
  416. */
  417. return __iosapic_read(addr, IOSAPIC_VERSION);
  418. }
  419. static int iosapic_find_sharable_irq(unsigned long trigger, unsigned long pol)
  420. {
  421. int i, irq = -ENOSPC, min_count = -1;
  422. struct iosapic_intr_info *info;
  423. /*
  424. * shared vectors for edge-triggered interrupts are not
  425. * supported yet
  426. */
  427. if (trigger == IOSAPIC_EDGE)
  428. return -EINVAL;
  429. for (i = 0; i < NR_IRQS; i++) {
  430. info = &iosapic_intr_info[i];
  431. if (info->trigger == trigger && info->polarity == pol &&
  432. (info->dmode == IOSAPIC_FIXED ||
  433. info->dmode == IOSAPIC_LOWEST_PRIORITY) &&
  434. can_request_irq(i, IRQF_SHARED)) {
  435. if (min_count == -1 || info->count < min_count) {
  436. irq = i;
  437. min_count = info->count;
  438. }
  439. }
  440. }
  441. return irq;
  442. }
  443. /*
  444. * if the given vector is already owned by other,
  445. * assign a new vector for the other and make the vector available
  446. */
  447. static void __init
  448. iosapic_reassign_vector (int irq)
  449. {
  450. int new_irq;
  451. if (iosapic_intr_info[irq].count) {
  452. new_irq = create_irq();
  453. if (new_irq < 0)
  454. panic("%s: out of interrupt vectors!\n", __func__);
  455. printk(KERN_INFO "Reassigning vector %d to %d\n",
  456. irq_to_vector(irq), irq_to_vector(new_irq));
  457. memcpy(&iosapic_intr_info[new_irq], &iosapic_intr_info[irq],
  458. sizeof(struct iosapic_intr_info));
  459. INIT_LIST_HEAD(&iosapic_intr_info[new_irq].rtes);
  460. list_move(iosapic_intr_info[irq].rtes.next,
  461. &iosapic_intr_info[new_irq].rtes);
  462. memset(&iosapic_intr_info[irq], 0,
  463. sizeof(struct iosapic_intr_info));
  464. iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
  465. INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
  466. }
  467. }
  468. static inline int irq_is_shared (int irq)
  469. {
  470. return (iosapic_intr_info[irq].count > 1);
  471. }
  472. struct irq_chip*
  473. ia64_native_iosapic_get_irq_chip(unsigned long trigger)
  474. {
  475. if (trigger == IOSAPIC_EDGE)
  476. return &irq_type_iosapic_edge;
  477. else
  478. return &irq_type_iosapic_level;
  479. }
  480. static int
  481. register_intr (unsigned int gsi, int irq, unsigned char delivery,
  482. unsigned long polarity, unsigned long trigger)
  483. {
  484. struct irq_chip *chip, *irq_type;
  485. int index;
  486. struct iosapic_rte_info *rte;
  487. index = find_iosapic(gsi);
  488. if (index < 0) {
  489. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  490. __func__, gsi);
  491. return -ENODEV;
  492. }
  493. rte = find_rte(irq, gsi);
  494. if (!rte) {
  495. rte = kzalloc(sizeof (*rte), GFP_ATOMIC);
  496. if (!rte) {
  497. printk(KERN_WARNING "%s: cannot allocate memory\n",
  498. __func__);
  499. return -ENOMEM;
  500. }
  501. rte->iosapic = &iosapic_lists[index];
  502. rte->rte_index = gsi - rte->iosapic->gsi_base;
  503. rte->refcnt++;
  504. list_add_tail(&rte->rte_list, &iosapic_intr_info[irq].rtes);
  505. iosapic_intr_info[irq].count++;
  506. iosapic_lists[index].rtes_inuse++;
  507. }
  508. else if (rte->refcnt == NO_REF_RTE) {
  509. struct iosapic_intr_info *info = &iosapic_intr_info[irq];
  510. if (info->count > 0 &&
  511. (info->trigger != trigger || info->polarity != polarity)){
  512. printk (KERN_WARNING
  513. "%s: cannot override the interrupt\n",
  514. __func__);
  515. return -EINVAL;
  516. }
  517. rte->refcnt++;
  518. iosapic_intr_info[irq].count++;
  519. iosapic_lists[index].rtes_inuse++;
  520. }
  521. iosapic_intr_info[irq].polarity = polarity;
  522. iosapic_intr_info[irq].dmode = delivery;
  523. iosapic_intr_info[irq].trigger = trigger;
  524. irq_type = iosapic_get_irq_chip(trigger);
  525. chip = irq_get_chip(irq);
  526. if (irq_type != NULL && chip != irq_type) {
  527. if (chip != &no_irq_chip)
  528. printk(KERN_WARNING
  529. "%s: changing vector %d from %s to %s\n",
  530. __func__, irq_to_vector(irq),
  531. chip->name, irq_type->name);
  532. chip = irq_type;
  533. }
  534. __irq_set_chip_handler_name_locked(irq, chip, trigger == IOSAPIC_EDGE ?
  535. handle_edge_irq : handle_level_irq,
  536. NULL);
  537. return 0;
  538. }
  539. static unsigned int
  540. get_target_cpu (unsigned int gsi, int irq)
  541. {
  542. #ifdef CONFIG_SMP
  543. static int cpu = -1;
  544. extern int cpe_vector;
  545. cpumask_t domain = irq_to_domain(irq);
  546. /*
  547. * In case of vector shared by multiple RTEs, all RTEs that
  548. * share the vector need to use the same destination CPU.
  549. */
  550. if (iosapic_intr_info[irq].count)
  551. return iosapic_intr_info[irq].dest;
  552. /*
  553. * If the platform supports redirection via XTP, let it
  554. * distribute interrupts.
  555. */
  556. if (smp_int_redirect & SMP_IRQ_REDIRECTION)
  557. return cpu_physical_id(smp_processor_id());
  558. /*
  559. * Some interrupts (ACPI SCI, for instance) are registered
  560. * before the BSP is marked as online.
  561. */
  562. if (!cpu_online(smp_processor_id()))
  563. return cpu_physical_id(smp_processor_id());
  564. #ifdef CONFIG_ACPI
  565. if (cpe_vector > 0 && irq_to_vector(irq) == IA64_CPEP_VECTOR)
  566. return get_cpei_target_cpu();
  567. #endif
  568. #ifdef CONFIG_NUMA
  569. {
  570. int num_cpus, cpu_index, iosapic_index, numa_cpu, i = 0;
  571. const struct cpumask *cpu_mask;
  572. iosapic_index = find_iosapic(gsi);
  573. if (iosapic_index < 0 ||
  574. iosapic_lists[iosapic_index].node == MAX_NUMNODES)
  575. goto skip_numa_setup;
  576. cpu_mask = cpumask_of_node(iosapic_lists[iosapic_index].node);
  577. num_cpus = 0;
  578. for_each_cpu_and(numa_cpu, cpu_mask, &domain) {
  579. if (cpu_online(numa_cpu))
  580. num_cpus++;
  581. }
  582. if (!num_cpus)
  583. goto skip_numa_setup;
  584. /* Use irq assignment to distribute across cpus in node */
  585. cpu_index = irq % num_cpus;
  586. for_each_cpu_and(numa_cpu, cpu_mask, &domain)
  587. if (cpu_online(numa_cpu) && i++ >= cpu_index)
  588. break;
  589. if (numa_cpu < nr_cpu_ids)
  590. return cpu_physical_id(numa_cpu);
  591. }
  592. skip_numa_setup:
  593. #endif
  594. /*
  595. * Otherwise, round-robin interrupt vectors across all the
  596. * processors. (It'd be nice if we could be smarter in the
  597. * case of NUMA.)
  598. */
  599. do {
  600. if (++cpu >= nr_cpu_ids)
  601. cpu = 0;
  602. } while (!cpu_online(cpu) || !cpu_isset(cpu, domain));
  603. return cpu_physical_id(cpu);
  604. #else /* CONFIG_SMP */
  605. return cpu_physical_id(smp_processor_id());
  606. #endif
  607. }
  608. static inline unsigned char choose_dmode(void)
  609. {
  610. #ifdef CONFIG_SMP
  611. if (smp_int_redirect & SMP_IRQ_REDIRECTION)
  612. return IOSAPIC_LOWEST_PRIORITY;
  613. #endif
  614. return IOSAPIC_FIXED;
  615. }
  616. /*
  617. * ACPI can describe IOSAPIC interrupts via static tables and namespace
  618. * methods. This provides an interface to register those interrupts and
  619. * program the IOSAPIC RTE.
  620. */
  621. int
  622. iosapic_register_intr (unsigned int gsi,
  623. unsigned long polarity, unsigned long trigger)
  624. {
  625. int irq, mask = 1, err;
  626. unsigned int dest;
  627. unsigned long flags;
  628. struct iosapic_rte_info *rte;
  629. u32 low32;
  630. unsigned char dmode;
  631. struct irq_desc *desc;
  632. /*
  633. * If this GSI has already been registered (i.e., it's a
  634. * shared interrupt, or we lost a race to register it),
  635. * don't touch the RTE.
  636. */
  637. spin_lock_irqsave(&iosapic_lock, flags);
  638. irq = __gsi_to_irq(gsi);
  639. if (irq > 0) {
  640. rte = find_rte(irq, gsi);
  641. if(iosapic_intr_info[irq].count == 0) {
  642. assign_irq_vector(irq);
  643. dynamic_irq_init(irq);
  644. } else if (rte->refcnt != NO_REF_RTE) {
  645. rte->refcnt++;
  646. goto unlock_iosapic_lock;
  647. }
  648. } else
  649. irq = create_irq();
  650. /* If vector is running out, we try to find a sharable vector */
  651. if (irq < 0) {
  652. irq = iosapic_find_sharable_irq(trigger, polarity);
  653. if (irq < 0)
  654. goto unlock_iosapic_lock;
  655. }
  656. desc = irq_to_desc(irq);
  657. raw_spin_lock(&desc->lock);
  658. dest = get_target_cpu(gsi, irq);
  659. dmode = choose_dmode();
  660. err = register_intr(gsi, irq, dmode, polarity, trigger);
  661. if (err < 0) {
  662. raw_spin_unlock(&desc->lock);
  663. irq = err;
  664. goto unlock_iosapic_lock;
  665. }
  666. /*
  667. * If the vector is shared and already unmasked for other
  668. * interrupt sources, don't mask it.
  669. */
  670. low32 = iosapic_intr_info[irq].low32;
  671. if (irq_is_shared(irq) && !(low32 & IOSAPIC_MASK))
  672. mask = 0;
  673. set_rte(gsi, irq, dest, mask);
  674. printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n",
  675. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  676. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  677. cpu_logical_id(dest), dest, irq_to_vector(irq));
  678. raw_spin_unlock(&desc->lock);
  679. unlock_iosapic_lock:
  680. spin_unlock_irqrestore(&iosapic_lock, flags);
  681. return irq;
  682. }
  683. void
  684. iosapic_unregister_intr (unsigned int gsi)
  685. {
  686. unsigned long flags;
  687. int irq, index;
  688. u32 low32;
  689. unsigned long trigger, polarity;
  690. unsigned int dest;
  691. struct iosapic_rte_info *rte;
  692. /*
  693. * If the irq associated with the gsi is not found,
  694. * iosapic_unregister_intr() is unbalanced. We need to check
  695. * this again after getting locks.
  696. */
  697. irq = gsi_to_irq(gsi);
  698. if (irq < 0) {
  699. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  700. gsi);
  701. WARN_ON(1);
  702. return;
  703. }
  704. spin_lock_irqsave(&iosapic_lock, flags);
  705. if ((rte = find_rte(irq, gsi)) == NULL) {
  706. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  707. gsi);
  708. WARN_ON(1);
  709. goto out;
  710. }
  711. if (--rte->refcnt > 0)
  712. goto out;
  713. rte->refcnt = NO_REF_RTE;
  714. /* Mask the interrupt */
  715. low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK;
  716. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32);
  717. iosapic_intr_info[irq].count--;
  718. index = find_iosapic(gsi);
  719. iosapic_lists[index].rtes_inuse--;
  720. WARN_ON(iosapic_lists[index].rtes_inuse < 0);
  721. trigger = iosapic_intr_info[irq].trigger;
  722. polarity = iosapic_intr_info[irq].polarity;
  723. dest = iosapic_intr_info[irq].dest;
  724. printk(KERN_INFO
  725. "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n",
  726. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  727. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  728. cpu_logical_id(dest), dest, irq_to_vector(irq));
  729. if (iosapic_intr_info[irq].count == 0) {
  730. #ifdef CONFIG_SMP
  731. /* Clear affinity */
  732. cpumask_setall(irq_get_irq_data(irq)->affinity);
  733. #endif
  734. /* Clear the interrupt information */
  735. iosapic_intr_info[irq].dest = 0;
  736. iosapic_intr_info[irq].dmode = 0;
  737. iosapic_intr_info[irq].polarity = 0;
  738. iosapic_intr_info[irq].trigger = 0;
  739. iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
  740. /* Destroy and reserve IRQ */
  741. destroy_and_reserve_irq(irq);
  742. }
  743. out:
  744. spin_unlock_irqrestore(&iosapic_lock, flags);
  745. }
  746. /*
  747. * ACPI calls this when it finds an entry for a platform interrupt.
  748. */
  749. int __init
  750. iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
  751. int iosapic_vector, u16 eid, u16 id,
  752. unsigned long polarity, unsigned long trigger)
  753. {
  754. static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"};
  755. unsigned char delivery;
  756. int irq, vector, mask = 0;
  757. unsigned int dest = ((id << 8) | eid) & 0xffff;
  758. switch (int_type) {
  759. case ACPI_INTERRUPT_PMI:
  760. irq = vector = iosapic_vector;
  761. bind_irq_vector(irq, vector, CPU_MASK_ALL);
  762. /*
  763. * since PMI vector is alloc'd by FW(ACPI) not by kernel,
  764. * we need to make sure the vector is available
  765. */
  766. iosapic_reassign_vector(irq);
  767. delivery = IOSAPIC_PMI;
  768. break;
  769. case ACPI_INTERRUPT_INIT:
  770. irq = create_irq();
  771. if (irq < 0)
  772. panic("%s: out of interrupt vectors!\n", __func__);
  773. vector = irq_to_vector(irq);
  774. delivery = IOSAPIC_INIT;
  775. break;
  776. case ACPI_INTERRUPT_CPEI:
  777. irq = vector = IA64_CPE_VECTOR;
  778. BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
  779. delivery = IOSAPIC_FIXED;
  780. mask = 1;
  781. break;
  782. default:
  783. printk(KERN_ERR "%s: invalid int type 0x%x\n", __func__,
  784. int_type);
  785. return -1;
  786. }
  787. register_intr(gsi, irq, delivery, polarity, trigger);
  788. printk(KERN_INFO
  789. "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)"
  790. " vector %d\n",
  791. int_type < ARRAY_SIZE(name) ? name[int_type] : "unknown",
  792. int_type, gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  793. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  794. cpu_logical_id(dest), dest, vector);
  795. set_rte(gsi, irq, dest, mask);
  796. return vector;
  797. }
  798. /*
  799. * ACPI calls this when it finds an entry for a legacy ISA IRQ override.
  800. */
  801. void __devinit
  802. iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi,
  803. unsigned long polarity,
  804. unsigned long trigger)
  805. {
  806. int vector, irq;
  807. unsigned int dest = cpu_physical_id(smp_processor_id());
  808. unsigned char dmode;
  809. irq = vector = isa_irq_to_vector(isa_irq);
  810. BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
  811. dmode = choose_dmode();
  812. register_intr(gsi, irq, dmode, polarity, trigger);
  813. DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n",
  814. isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level",
  815. polarity == IOSAPIC_POL_HIGH ? "high" : "low",
  816. cpu_logical_id(dest), dest, vector);
  817. set_rte(gsi, irq, dest, 1);
  818. }
  819. void __init
  820. ia64_native_iosapic_pcat_compat_init(void)
  821. {
  822. if (pcat_compat) {
  823. /*
  824. * Disable the compatibility mode interrupts (8259 style),
  825. * needs IN/OUT support enabled.
  826. */
  827. printk(KERN_INFO
  828. "%s: Disabling PC-AT compatible 8259 interrupts\n",
  829. __func__);
  830. outb(0xff, 0xA1);
  831. outb(0xff, 0x21);
  832. }
  833. }
  834. void __init
  835. iosapic_system_init (int system_pcat_compat)
  836. {
  837. int irq;
  838. for (irq = 0; irq < NR_IRQS; ++irq) {
  839. iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
  840. /* mark as unused */
  841. INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
  842. iosapic_intr_info[irq].count = 0;
  843. }
  844. pcat_compat = system_pcat_compat;
  845. if (pcat_compat)
  846. iosapic_pcat_compat_init();
  847. }
  848. static inline int
  849. iosapic_alloc (void)
  850. {
  851. int index;
  852. for (index = 0; index < NR_IOSAPICS; index++)
  853. if (!iosapic_lists[index].addr)
  854. return index;
  855. printk(KERN_WARNING "%s: failed to allocate iosapic\n", __func__);
  856. return -1;
  857. }
  858. static inline void
  859. iosapic_free (int index)
  860. {
  861. memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0]));
  862. }
  863. static inline int
  864. iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver)
  865. {
  866. int index;
  867. unsigned int gsi_end, base, end;
  868. /* check gsi range */
  869. gsi_end = gsi_base + ((ver >> 16) & 0xff);
  870. for (index = 0; index < NR_IOSAPICS; index++) {
  871. if (!iosapic_lists[index].addr)
  872. continue;
  873. base = iosapic_lists[index].gsi_base;
  874. end = base + iosapic_lists[index].num_rte - 1;
  875. if (gsi_end < base || end < gsi_base)
  876. continue; /* OK */
  877. return -EBUSY;
  878. }
  879. return 0;
  880. }
  881. int __devinit
  882. iosapic_init (unsigned long phys_addr, unsigned int gsi_base)
  883. {
  884. int num_rte, err, index;
  885. unsigned int isa_irq, ver;
  886. char __iomem *addr;
  887. unsigned long flags;
  888. spin_lock_irqsave(&iosapic_lock, flags);
  889. index = find_iosapic(gsi_base);
  890. if (index >= 0) {
  891. spin_unlock_irqrestore(&iosapic_lock, flags);
  892. return -EBUSY;
  893. }
  894. addr = ioremap(phys_addr, 0);
  895. if (addr == NULL) {
  896. spin_unlock_irqrestore(&iosapic_lock, flags);
  897. return -ENOMEM;
  898. }
  899. ver = iosapic_version(addr);
  900. if ((err = iosapic_check_gsi_range(gsi_base, ver))) {
  901. iounmap(addr);
  902. spin_unlock_irqrestore(&iosapic_lock, flags);
  903. return err;
  904. }
  905. /*
  906. * The MAX_REDIR register holds the highest input pin number
  907. * (starting from 0). We add 1 so that we can use it for
  908. * number of pins (= RTEs)
  909. */
  910. num_rte = ((ver >> 16) & 0xff) + 1;
  911. index = iosapic_alloc();
  912. iosapic_lists[index].addr = addr;
  913. iosapic_lists[index].gsi_base = gsi_base;
  914. iosapic_lists[index].num_rte = num_rte;
  915. #ifdef CONFIG_NUMA
  916. iosapic_lists[index].node = MAX_NUMNODES;
  917. #endif
  918. spin_lock_init(&iosapic_lists[index].lock);
  919. spin_unlock_irqrestore(&iosapic_lock, flags);
  920. if ((gsi_base == 0) && pcat_compat) {
  921. /*
  922. * Map the legacy ISA devices into the IOSAPIC data. Some of
  923. * these may get reprogrammed later on with data from the ACPI
  924. * Interrupt Source Override table.
  925. */
  926. for (isa_irq = 0; isa_irq < 16; ++isa_irq)
  927. iosapic_override_isa_irq(isa_irq, isa_irq,
  928. IOSAPIC_POL_HIGH,
  929. IOSAPIC_EDGE);
  930. }
  931. return 0;
  932. }
  933. #ifdef CONFIG_HOTPLUG
  934. int
  935. iosapic_remove (unsigned int gsi_base)
  936. {
  937. int index, err = 0;
  938. unsigned long flags;
  939. spin_lock_irqsave(&iosapic_lock, flags);
  940. index = find_iosapic(gsi_base);
  941. if (index < 0) {
  942. printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n",
  943. __func__, gsi_base);
  944. goto out;
  945. }
  946. if (iosapic_lists[index].rtes_inuse) {
  947. err = -EBUSY;
  948. printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n",
  949. __func__, gsi_base);
  950. goto out;
  951. }
  952. iounmap(iosapic_lists[index].addr);
  953. iosapic_free(index);
  954. out:
  955. spin_unlock_irqrestore(&iosapic_lock, flags);
  956. return err;
  957. }
  958. #endif /* CONFIG_HOTPLUG */
  959. #ifdef CONFIG_NUMA
  960. void __devinit
  961. map_iosapic_to_node(unsigned int gsi_base, int node)
  962. {
  963. int index;
  964. index = find_iosapic(gsi_base);
  965. if (index < 0) {
  966. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  967. __func__, gsi_base);
  968. return;
  969. }
  970. iosapic_lists[index].node = node;
  971. return;
  972. }
  973. #endif