intel_intr_remapping.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967
  1. #include <linux/interrupt.h>
  2. #include <linux/dmar.h>
  3. #include <linux/spinlock.h>
  4. #include <linux/slab.h>
  5. #include <linux/jiffies.h>
  6. #include <linux/hpet.h>
  7. #include <linux/pci.h>
  8. #include <linux/irq.h>
  9. #include <asm/io_apic.h>
  10. #include <asm/smp.h>
  11. #include <asm/cpu.h>
  12. #include <linux/intel-iommu.h>
  13. #include <acpi/acpi.h>
  14. #include <asm/intr_remapping.h>
  15. #include <asm/pci-direct.h>
  16. #include "intr_remapping.h"
  17. struct ioapic_scope {
  18. struct intel_iommu *iommu;
  19. unsigned int id;
  20. unsigned int bus; /* PCI bus number */
  21. unsigned int devfn; /* PCI devfn number */
  22. };
  23. struct hpet_scope {
  24. struct intel_iommu *iommu;
  25. u8 id;
  26. unsigned int bus;
  27. unsigned int devfn;
  28. };
  29. #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
  30. #define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
  31. static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
  32. static struct hpet_scope ir_hpet[MAX_HPET_TBS];
  33. static int ir_ioapic_num, ir_hpet_num;
  34. static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
  35. static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
  36. {
  37. struct irq_cfg *cfg = irq_get_chip_data(irq);
  38. return cfg ? &cfg->irq_2_iommu : NULL;
  39. }
  40. int get_irte(int irq, struct irte *entry)
  41. {
  42. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  43. unsigned long flags;
  44. int index;
  45. if (!entry || !irq_iommu)
  46. return -1;
  47. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  48. index = irq_iommu->irte_index + irq_iommu->sub_handle;
  49. *entry = *(irq_iommu->iommu->ir_table->base + index);
  50. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  51. return 0;
  52. }
  53. int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
  54. {
  55. struct ir_table *table = iommu->ir_table;
  56. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  57. u16 index, start_index;
  58. unsigned int mask = 0;
  59. unsigned long flags;
  60. int i;
  61. if (!count || !irq_iommu)
  62. return -1;
  63. /*
  64. * start the IRTE search from index 0.
  65. */
  66. index = start_index = 0;
  67. if (count > 1) {
  68. count = __roundup_pow_of_two(count);
  69. mask = ilog2(count);
  70. }
  71. if (mask > ecap_max_handle_mask(iommu->ecap)) {
  72. printk(KERN_ERR
  73. "Requested mask %x exceeds the max invalidation handle"
  74. " mask value %Lx\n", mask,
  75. ecap_max_handle_mask(iommu->ecap));
  76. return -1;
  77. }
  78. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  79. do {
  80. for (i = index; i < index + count; i++)
  81. if (table->base[i].present)
  82. break;
  83. /* empty index found */
  84. if (i == index + count)
  85. break;
  86. index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
  87. if (index == start_index) {
  88. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  89. printk(KERN_ERR "can't allocate an IRTE\n");
  90. return -1;
  91. }
  92. } while (1);
  93. for (i = index; i < index + count; i++)
  94. table->base[i].present = 1;
  95. irq_iommu->iommu = iommu;
  96. irq_iommu->irte_index = index;
  97. irq_iommu->sub_handle = 0;
  98. irq_iommu->irte_mask = mask;
  99. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  100. return index;
  101. }
  102. static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
  103. {
  104. struct qi_desc desc;
  105. desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
  106. | QI_IEC_SELECTIVE;
  107. desc.high = 0;
  108. return qi_submit_sync(&desc, iommu);
  109. }
  110. int map_irq_to_irte_handle(int irq, u16 *sub_handle)
  111. {
  112. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  113. unsigned long flags;
  114. int index;
  115. if (!irq_iommu)
  116. return -1;
  117. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  118. *sub_handle = irq_iommu->sub_handle;
  119. index = irq_iommu->irte_index;
  120. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  121. return index;
  122. }
  123. int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
  124. {
  125. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  126. unsigned long flags;
  127. if (!irq_iommu)
  128. return -1;
  129. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  130. irq_iommu->iommu = iommu;
  131. irq_iommu->irte_index = index;
  132. irq_iommu->sub_handle = subhandle;
  133. irq_iommu->irte_mask = 0;
  134. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  135. return 0;
  136. }
  137. int modify_irte(int irq, struct irte *irte_modified)
  138. {
  139. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  140. struct intel_iommu *iommu;
  141. unsigned long flags;
  142. struct irte *irte;
  143. int rc, index;
  144. if (!irq_iommu)
  145. return -1;
  146. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  147. iommu = irq_iommu->iommu;
  148. index = irq_iommu->irte_index + irq_iommu->sub_handle;
  149. irte = &iommu->ir_table->base[index];
  150. set_64bit(&irte->low, irte_modified->low);
  151. set_64bit(&irte->high, irte_modified->high);
  152. __iommu_flush_cache(iommu, irte, sizeof(*irte));
  153. rc = qi_flush_iec(iommu, index, 0);
  154. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  155. return rc;
  156. }
  157. struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
  158. {
  159. int i;
  160. for (i = 0; i < MAX_HPET_TBS; i++)
  161. if (ir_hpet[i].id == hpet_id)
  162. return ir_hpet[i].iommu;
  163. return NULL;
  164. }
  165. struct intel_iommu *map_ioapic_to_ir(int apic)
  166. {
  167. int i;
  168. for (i = 0; i < MAX_IO_APICS; i++)
  169. if (ir_ioapic[i].id == apic)
  170. return ir_ioapic[i].iommu;
  171. return NULL;
  172. }
  173. struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
  174. {
  175. struct dmar_drhd_unit *drhd;
  176. drhd = dmar_find_matched_drhd_unit(dev);
  177. if (!drhd)
  178. return NULL;
  179. return drhd->iommu;
  180. }
  181. static int clear_entries(struct irq_2_iommu *irq_iommu)
  182. {
  183. struct irte *start, *entry, *end;
  184. struct intel_iommu *iommu;
  185. int index;
  186. if (irq_iommu->sub_handle)
  187. return 0;
  188. iommu = irq_iommu->iommu;
  189. index = irq_iommu->irte_index + irq_iommu->sub_handle;
  190. start = iommu->ir_table->base + index;
  191. end = start + (1 << irq_iommu->irte_mask);
  192. for (entry = start; entry < end; entry++) {
  193. set_64bit(&entry->low, 0);
  194. set_64bit(&entry->high, 0);
  195. }
  196. return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
  197. }
  198. int free_irte(int irq)
  199. {
  200. struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
  201. unsigned long flags;
  202. int rc;
  203. if (!irq_iommu)
  204. return -1;
  205. raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
  206. rc = clear_entries(irq_iommu);
  207. irq_iommu->iommu = NULL;
  208. irq_iommu->irte_index = 0;
  209. irq_iommu->sub_handle = 0;
  210. irq_iommu->irte_mask = 0;
  211. raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
  212. return rc;
  213. }
  214. /*
  215. * source validation type
  216. */
  217. #define SVT_NO_VERIFY 0x0 /* no verification is required */
  218. #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
  219. #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
  220. /*
  221. * source-id qualifier
  222. */
  223. #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
  224. #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
  225. * the third least significant bit
  226. */
  227. #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
  228. * the second and third least significant bits
  229. */
  230. #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
  231. * the least three significant bits
  232. */
  233. /*
  234. * set SVT, SQ and SID fields of irte to verify
  235. * source ids of interrupt requests
  236. */
  237. static void set_irte_sid(struct irte *irte, unsigned int svt,
  238. unsigned int sq, unsigned int sid)
  239. {
  240. if (disable_sourceid_checking)
  241. svt = SVT_NO_VERIFY;
  242. irte->svt = svt;
  243. irte->sq = sq;
  244. irte->sid = sid;
  245. }
  246. int set_ioapic_sid(struct irte *irte, int apic)
  247. {
  248. int i;
  249. u16 sid = 0;
  250. if (!irte)
  251. return -1;
  252. for (i = 0; i < MAX_IO_APICS; i++) {
  253. if (ir_ioapic[i].id == apic) {
  254. sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
  255. break;
  256. }
  257. }
  258. if (sid == 0) {
  259. pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
  260. return -1;
  261. }
  262. set_irte_sid(irte, 1, 0, sid);
  263. return 0;
  264. }
  265. int set_hpet_sid(struct irte *irte, u8 id)
  266. {
  267. int i;
  268. u16 sid = 0;
  269. if (!irte)
  270. return -1;
  271. for (i = 0; i < MAX_HPET_TBS; i++) {
  272. if (ir_hpet[i].id == id) {
  273. sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
  274. break;
  275. }
  276. }
  277. if (sid == 0) {
  278. pr_warning("Failed to set source-id of HPET block (%d)\n", id);
  279. return -1;
  280. }
  281. /*
  282. * Should really use SQ_ALL_16. Some platforms are broken.
  283. * While we figure out the right quirks for these broken platforms, use
  284. * SQ_13_IGNORE_3 for now.
  285. */
  286. set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);
  287. return 0;
  288. }
  289. int set_msi_sid(struct irte *irte, struct pci_dev *dev)
  290. {
  291. struct pci_dev *bridge;
  292. if (!irte || !dev)
  293. return -1;
  294. /* PCIe device or Root Complex integrated PCI device */
  295. if (pci_is_pcie(dev) || !dev->bus->parent) {
  296. set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
  297. (dev->bus->number << 8) | dev->devfn);
  298. return 0;
  299. }
  300. bridge = pci_find_upstream_pcie_bridge(dev);
  301. if (bridge) {
  302. if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */
  303. set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
  304. (bridge->bus->number << 8) | dev->bus->number);
  305. else /* this is a legacy PCI bridge */
  306. set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
  307. (bridge->bus->number << 8) | bridge->devfn);
  308. }
  309. return 0;
  310. }
  311. static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
  312. {
  313. u64 addr;
  314. u32 sts;
  315. unsigned long flags;
  316. addr = virt_to_phys((void *)iommu->ir_table->base);
  317. raw_spin_lock_irqsave(&iommu->register_lock, flags);
  318. dmar_writeq(iommu->reg + DMAR_IRTA_REG,
  319. (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
  320. /* Set interrupt-remapping table pointer */
  321. iommu->gcmd |= DMA_GCMD_SIRTP;
  322. writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
  323. IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
  324. readl, (sts & DMA_GSTS_IRTPS), sts);
  325. raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
  326. /*
  327. * global invalidation of interrupt entry cache before enabling
  328. * interrupt-remapping.
  329. */
  330. qi_global_iec(iommu);
  331. raw_spin_lock_irqsave(&iommu->register_lock, flags);
  332. /* Enable interrupt-remapping */
  333. iommu->gcmd |= DMA_GCMD_IRE;
  334. writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
  335. IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
  336. readl, (sts & DMA_GSTS_IRES), sts);
  337. raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
  338. }
  339. static int intel_setup_intr_remapping(struct intel_iommu *iommu, int mode)
  340. {
  341. struct ir_table *ir_table;
  342. struct page *pages;
  343. ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
  344. GFP_ATOMIC);
  345. if (!iommu->ir_table)
  346. return -ENOMEM;
  347. pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
  348. INTR_REMAP_PAGE_ORDER);
  349. if (!pages) {
  350. printk(KERN_ERR "failed to allocate pages of order %d\n",
  351. INTR_REMAP_PAGE_ORDER);
  352. kfree(iommu->ir_table);
  353. return -ENOMEM;
  354. }
  355. ir_table->base = page_address(pages);
  356. iommu_set_intr_remapping(iommu, mode);
  357. return 0;
  358. }
  359. /*
  360. * Disable Interrupt Remapping.
  361. */
  362. static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
  363. {
  364. unsigned long flags;
  365. u32 sts;
  366. if (!ecap_ir_support(iommu->ecap))
  367. return;
  368. /*
  369. * global invalidation of interrupt entry cache before disabling
  370. * interrupt-remapping.
  371. */
  372. qi_global_iec(iommu);
  373. raw_spin_lock_irqsave(&iommu->register_lock, flags);
  374. sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
  375. if (!(sts & DMA_GSTS_IRES))
  376. goto end;
  377. iommu->gcmd &= ~DMA_GCMD_IRE;
  378. writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
  379. IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
  380. readl, !(sts & DMA_GSTS_IRES), sts);
  381. end:
  382. raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
  383. }
  384. static int __init dmar_x2apic_optout(void)
  385. {
  386. struct acpi_table_dmar *dmar;
  387. dmar = (struct acpi_table_dmar *)dmar_tbl;
  388. if (!dmar || no_x2apic_optout)
  389. return 0;
  390. return dmar->flags & DMAR_X2APIC_OPT_OUT;
  391. }
  392. static int __init intel_intr_remapping_supported(void)
  393. {
  394. struct dmar_drhd_unit *drhd;
  395. if (disable_intremap)
  396. return 0;
  397. if (!dmar_ir_support())
  398. return 0;
  399. for_each_drhd_unit(drhd) {
  400. struct intel_iommu *iommu = drhd->iommu;
  401. if (!ecap_ir_support(iommu->ecap))
  402. return 0;
  403. }
  404. return 1;
  405. }
  406. static int __init intel_enable_intr_remapping(void)
  407. {
  408. struct dmar_drhd_unit *drhd;
  409. int setup = 0;
  410. int eim = 0;
  411. if (parse_ioapics_under_ir() != 1) {
  412. printk(KERN_INFO "Not enable interrupt remapping\n");
  413. return -1;
  414. }
  415. if (x2apic_supported()) {
  416. eim = !dmar_x2apic_optout();
  417. WARN(!eim, KERN_WARNING
  418. "Your BIOS is broken and requested that x2apic be disabled\n"
  419. "This will leave your machine vulnerable to irq-injection attacks\n"
  420. "Use 'intremap=no_x2apic_optout' to override BIOS request\n");
  421. }
  422. for_each_drhd_unit(drhd) {
  423. struct intel_iommu *iommu = drhd->iommu;
  424. /*
  425. * If the queued invalidation is already initialized,
  426. * shouldn't disable it.
  427. */
  428. if (iommu->qi)
  429. continue;
  430. /*
  431. * Clear previous faults.
  432. */
  433. dmar_fault(-1, iommu);
  434. /*
  435. * Disable intr remapping and queued invalidation, if already
  436. * enabled prior to OS handover.
  437. */
  438. iommu_disable_intr_remapping(iommu);
  439. dmar_disable_qi(iommu);
  440. }
  441. /*
  442. * check for the Interrupt-remapping support
  443. */
  444. for_each_drhd_unit(drhd) {
  445. struct intel_iommu *iommu = drhd->iommu;
  446. if (!ecap_ir_support(iommu->ecap))
  447. continue;
  448. if (eim && !ecap_eim_support(iommu->ecap)) {
  449. printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
  450. " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
  451. return -1;
  452. }
  453. }
  454. /*
  455. * Enable queued invalidation for all the DRHD's.
  456. */
  457. for_each_drhd_unit(drhd) {
  458. int ret;
  459. struct intel_iommu *iommu = drhd->iommu;
  460. ret = dmar_enable_qi(iommu);
  461. if (ret) {
  462. printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
  463. " invalidation, ecap %Lx, ret %d\n",
  464. drhd->reg_base_addr, iommu->ecap, ret);
  465. return -1;
  466. }
  467. }
  468. /*
  469. * Setup Interrupt-remapping for all the DRHD's now.
  470. */
  471. for_each_drhd_unit(drhd) {
  472. struct intel_iommu *iommu = drhd->iommu;
  473. if (!ecap_ir_support(iommu->ecap))
  474. continue;
  475. if (intel_setup_intr_remapping(iommu, eim))
  476. goto error;
  477. setup = 1;
  478. }
  479. if (!setup)
  480. goto error;
  481. intr_remapping_enabled = 1;
  482. pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
  483. return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
  484. error:
  485. /*
  486. * handle error condition gracefully here!
  487. */
  488. return -1;
  489. }
  490. static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
  491. struct intel_iommu *iommu)
  492. {
  493. struct acpi_dmar_pci_path *path;
  494. u8 bus;
  495. int count;
  496. bus = scope->bus;
  497. path = (struct acpi_dmar_pci_path *)(scope + 1);
  498. count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  499. / sizeof(struct acpi_dmar_pci_path);
  500. while (--count > 0) {
  501. /*
  502. * Access PCI directly due to the PCI
  503. * subsystem isn't initialized yet.
  504. */
  505. bus = read_pci_config_byte(bus, path->dev, path->fn,
  506. PCI_SECONDARY_BUS);
  507. path++;
  508. }
  509. ir_hpet[ir_hpet_num].bus = bus;
  510. ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn);
  511. ir_hpet[ir_hpet_num].iommu = iommu;
  512. ir_hpet[ir_hpet_num].id = scope->enumeration_id;
  513. ir_hpet_num++;
  514. }
  515. static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
  516. struct intel_iommu *iommu)
  517. {
  518. struct acpi_dmar_pci_path *path;
  519. u8 bus;
  520. int count;
  521. bus = scope->bus;
  522. path = (struct acpi_dmar_pci_path *)(scope + 1);
  523. count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  524. / sizeof(struct acpi_dmar_pci_path);
  525. while (--count > 0) {
  526. /*
  527. * Access PCI directly due to the PCI
  528. * subsystem isn't initialized yet.
  529. */
  530. bus = read_pci_config_byte(bus, path->dev, path->fn,
  531. PCI_SECONDARY_BUS);
  532. path++;
  533. }
  534. ir_ioapic[ir_ioapic_num].bus = bus;
  535. ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn);
  536. ir_ioapic[ir_ioapic_num].iommu = iommu;
  537. ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
  538. ir_ioapic_num++;
  539. }
  540. static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
  541. struct intel_iommu *iommu)
  542. {
  543. struct acpi_dmar_hardware_unit *drhd;
  544. struct acpi_dmar_device_scope *scope;
  545. void *start, *end;
  546. drhd = (struct acpi_dmar_hardware_unit *)header;
  547. start = (void *)(drhd + 1);
  548. end = ((void *)drhd) + header->length;
  549. while (start < end) {
  550. scope = start;
  551. if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
  552. if (ir_ioapic_num == MAX_IO_APICS) {
  553. printk(KERN_WARNING "Exceeded Max IO APICS\n");
  554. return -1;
  555. }
  556. printk(KERN_INFO "IOAPIC id %d under DRHD base "
  557. " 0x%Lx IOMMU %d\n", scope->enumeration_id,
  558. drhd->address, iommu->seq_id);
  559. ir_parse_one_ioapic_scope(scope, iommu);
  560. } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
  561. if (ir_hpet_num == MAX_HPET_TBS) {
  562. printk(KERN_WARNING "Exceeded Max HPET blocks\n");
  563. return -1;
  564. }
  565. printk(KERN_INFO "HPET id %d under DRHD base"
  566. " 0x%Lx\n", scope->enumeration_id,
  567. drhd->address);
  568. ir_parse_one_hpet_scope(scope, iommu);
  569. }
  570. start += scope->length;
  571. }
  572. return 0;
  573. }
  574. /*
  575. * Finds the assocaition between IOAPIC's and its Interrupt-remapping
  576. * hardware unit.
  577. */
  578. int __init parse_ioapics_under_ir(void)
  579. {
  580. struct dmar_drhd_unit *drhd;
  581. int ir_supported = 0;
  582. for_each_drhd_unit(drhd) {
  583. struct intel_iommu *iommu = drhd->iommu;
  584. if (ecap_ir_support(iommu->ecap)) {
  585. if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
  586. return -1;
  587. ir_supported = 1;
  588. }
  589. }
  590. if (ir_supported && ir_ioapic_num != nr_ioapics) {
  591. printk(KERN_WARNING
  592. "Not all IO-APIC's listed under remapping hardware\n");
  593. return -1;
  594. }
  595. return ir_supported;
  596. }
  597. int __init ir_dev_scope_init(void)
  598. {
  599. if (!intr_remapping_enabled)
  600. return 0;
  601. return dmar_dev_scope_init();
  602. }
  603. rootfs_initcall(ir_dev_scope_init);
  604. static void disable_intr_remapping(void)
  605. {
  606. struct dmar_drhd_unit *drhd;
  607. struct intel_iommu *iommu = NULL;
  608. /*
  609. * Disable Interrupt-remapping for all the DRHD's now.
  610. */
  611. for_each_iommu(iommu, drhd) {
  612. if (!ecap_ir_support(iommu->ecap))
  613. continue;
  614. iommu_disable_intr_remapping(iommu);
  615. }
  616. }
  617. static int reenable_intr_remapping(int eim)
  618. {
  619. struct dmar_drhd_unit *drhd;
  620. int setup = 0;
  621. struct intel_iommu *iommu = NULL;
  622. for_each_iommu(iommu, drhd)
  623. if (iommu->qi)
  624. dmar_reenable_qi(iommu);
  625. /*
  626. * Setup Interrupt-remapping for all the DRHD's now.
  627. */
  628. for_each_iommu(iommu, drhd) {
  629. if (!ecap_ir_support(iommu->ecap))
  630. continue;
  631. /* Set up interrupt remapping for iommu.*/
  632. iommu_set_intr_remapping(iommu, eim);
  633. setup = 1;
  634. }
  635. if (!setup)
  636. goto error;
  637. return 0;
  638. error:
  639. /*
  640. * handle error condition gracefully here!
  641. */
  642. return -1;
  643. }
  644. static void prepare_irte(struct irte *irte, int vector,
  645. unsigned int dest)
  646. {
  647. memset(irte, 0, sizeof(*irte));
  648. irte->present = 1;
  649. irte->dst_mode = apic->irq_dest_mode;
  650. /*
  651. * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
  652. * actual level or edge trigger will be setup in the IO-APIC
  653. * RTE. This will help simplify level triggered irq migration.
  654. * For more details, see the comments (in io_apic.c) explainig IO-APIC
  655. * irq migration in the presence of interrupt-remapping.
  656. */
  657. irte->trigger_mode = 0;
  658. irte->dlvry_mode = apic->irq_delivery_mode;
  659. irte->vector = vector;
  660. irte->dest_id = IRTE_DEST(dest);
  661. irte->redir_hint = 1;
  662. }
  663. static int intel_setup_ioapic_entry(int irq,
  664. struct IO_APIC_route_entry *route_entry,
  665. unsigned int destination, int vector,
  666. struct io_apic_irq_attr *attr)
  667. {
  668. int ioapic_id = mpc_ioapic_id(attr->ioapic);
  669. struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id);
  670. struct IR_IO_APIC_route_entry *entry;
  671. struct irte irte;
  672. int index;
  673. if (!iommu) {
  674. pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
  675. return -ENODEV;
  676. }
  677. entry = (struct IR_IO_APIC_route_entry *)route_entry;
  678. index = alloc_irte(iommu, irq, 1);
  679. if (index < 0) {
  680. pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id);
  681. return -ENOMEM;
  682. }
  683. prepare_irte(&irte, vector, destination);
  684. /* Set source-id of interrupt request */
  685. set_ioapic_sid(&irte, ioapic_id);
  686. modify_irte(irq, &irte);
  687. apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: "
  688. "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d "
  689. "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X "
  690. "Avail:%X Vector:%02X Dest:%08X "
  691. "SID:%04X SQ:%X SVT:%X)\n",
  692. attr->ioapic, irte.present, irte.fpd, irte.dst_mode,
  693. irte.redir_hint, irte.trigger_mode, irte.dlvry_mode,
  694. irte.avail, irte.vector, irte.dest_id,
  695. irte.sid, irte.sq, irte.svt);
  696. memset(entry, 0, sizeof(*entry));
  697. entry->index2 = (index >> 15) & 0x1;
  698. entry->zero = 0;
  699. entry->format = 1;
  700. entry->index = (index & 0x7fff);
  701. /*
  702. * IO-APIC RTE will be configured with virtual vector.
  703. * irq handler will do the explicit EOI to the io-apic.
  704. */
  705. entry->vector = attr->ioapic_pin;
  706. entry->mask = 0; /* enable IRQ */
  707. entry->trigger = attr->trigger;
  708. entry->polarity = attr->polarity;
  709. /* Mask level triggered irqs.
  710. * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
  711. */
  712. if (attr->trigger)
  713. entry->mask = 1;
  714. return 0;
  715. }
  716. /*
  717. * Migrate the IO-APIC irq in the presence of intr-remapping.
  718. *
  719. * For both level and edge triggered, irq migration is a simple atomic
  720. * update(of vector and cpu destination) of IRTE and flush the hardware cache.
  721. *
  722. * For level triggered, we eliminate the io-apic RTE modification (with the
  723. * updated vector information), by using a virtual vector (io-apic pin number).
  724. * Real vector that is used for interrupting cpu will be coming from
  725. * the interrupt-remapping table entry.
  726. *
  727. * As the migration is a simple atomic update of IRTE, the same mechanism
  728. * is used to migrate MSI irq's in the presence of interrupt-remapping.
  729. */
  730. static int
  731. intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
  732. bool force)
  733. {
  734. struct irq_cfg *cfg = data->chip_data;
  735. unsigned int dest, irq = data->irq;
  736. struct irte irte;
  737. if (!cpumask_intersects(mask, cpu_online_mask))
  738. return -EINVAL;
  739. if (get_irte(irq, &irte))
  740. return -EBUSY;
  741. if (assign_irq_vector(irq, cfg, mask))
  742. return -EBUSY;
  743. dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
  744. irte.vector = cfg->vector;
  745. irte.dest_id = IRTE_DEST(dest);
  746. /*
  747. * Atomically updates the IRTE with the new destination, vector
  748. * and flushes the interrupt entry cache.
  749. */
  750. modify_irte(irq, &irte);
  751. /*
  752. * After this point, all the interrupts will start arriving
  753. * at the new destination. So, time to cleanup the previous
  754. * vector allocation.
  755. */
  756. if (cfg->move_in_progress)
  757. send_cleanup_vector(cfg);
  758. cpumask_copy(data->affinity, mask);
  759. return 0;
  760. }
  761. struct irq_remap_ops intel_irq_remap_ops = {
  762. .supported = intel_intr_remapping_supported,
  763. .hardware_init = dmar_table_init,
  764. .hardware_enable = intel_enable_intr_remapping,
  765. .hardware_disable = disable_intr_remapping,
  766. .hardware_reenable = reenable_intr_remapping,
  767. .enable_faulting = enable_drhd_fault_handling,
  768. .setup_ioapic_entry = intel_setup_ioapic_entry,
  769. .set_affinity = intel_ioapic_set_affinity,
  770. };