dmar.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814
  1. /*
  2. * Copyright (c) 2006, Intel Corporation.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms and conditions of the GNU General Public License,
  6. * version 2, as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope it will be useful, but WITHOUT
  9. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  11. * more details.
  12. *
  13. * You should have received a copy of the GNU General Public License along with
  14. * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15. * Place - Suite 330, Boston, MA 02111-1307 USA.
  16. *
  17. * Copyright (C) 2006-2008 Intel Corporation
  18. * Author: Ashok Raj <ashok.raj@intel.com>
  19. * Author: Shaohua Li <shaohua.li@intel.com>
  20. * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21. *
  22. * This file implements early detection/parsing of Remapping Devices
  23. * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24. * tables.
  25. *
  26. * These routines are used by both DMA-remapping and Interrupt-remapping
  27. */
  28. #include <linux/pci.h>
  29. #include <linux/dmar.h>
  30. #include <linux/iova.h>
  31. #include <linux/intel-iommu.h>
  32. #include <linux/timer.h>
  33. #undef PREFIX
  34. #define PREFIX "DMAR:"
  35. /* No locks are needed as DMA remapping hardware unit
  36. * list is constructed at boot time and hotplug of
  37. * these units are not supported by the architecture.
  38. */
  39. LIST_HEAD(dmar_drhd_units);
  40. static struct acpi_table_header * __initdata dmar_tbl;
  41. static acpi_size dmar_tbl_size;
  42. static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  43. {
  44. /*
  45. * add INCLUDE_ALL at the tail, so scan the list will find it at
  46. * the very end.
  47. */
  48. if (drhd->include_all)
  49. list_add_tail(&drhd->list, &dmar_drhd_units);
  50. else
  51. list_add(&drhd->list, &dmar_drhd_units);
  52. }
  53. static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  54. struct pci_dev **dev, u16 segment)
  55. {
  56. struct pci_bus *bus;
  57. struct pci_dev *pdev = NULL;
  58. struct acpi_dmar_pci_path *path;
  59. int count;
  60. bus = pci_find_bus(segment, scope->bus);
  61. path = (struct acpi_dmar_pci_path *)(scope + 1);
  62. count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  63. / sizeof(struct acpi_dmar_pci_path);
  64. while (count) {
  65. if (pdev)
  66. pci_dev_put(pdev);
  67. /*
  68. * Some BIOSes list non-exist devices in DMAR table, just
  69. * ignore it
  70. */
  71. if (!bus) {
  72. printk(KERN_WARNING
  73. PREFIX "Device scope bus [%d] not found\n",
  74. scope->bus);
  75. break;
  76. }
  77. pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  78. if (!pdev) {
  79. printk(KERN_WARNING PREFIX
  80. "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  81. segment, bus->number, path->dev, path->fn);
  82. break;
  83. }
  84. path ++;
  85. count --;
  86. bus = pdev->subordinate;
  87. }
  88. if (!pdev) {
  89. printk(KERN_WARNING PREFIX
  90. "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  91. segment, scope->bus, path->dev, path->fn);
  92. *dev = NULL;
  93. return 0;
  94. }
  95. if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
  96. pdev->subordinate) || (scope->entry_type == \
  97. ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
  98. pci_dev_put(pdev);
  99. printk(KERN_WARNING PREFIX
  100. "Device scope type does not match for %s\n",
  101. pci_name(pdev));
  102. return -EINVAL;
  103. }
  104. *dev = pdev;
  105. return 0;
  106. }
  107. static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
  108. struct pci_dev ***devices, u16 segment)
  109. {
  110. struct acpi_dmar_device_scope *scope;
  111. void * tmp = start;
  112. int index;
  113. int ret;
  114. *cnt = 0;
  115. while (start < end) {
  116. scope = start;
  117. if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  118. scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  119. (*cnt)++;
  120. else
  121. printk(KERN_WARNING PREFIX
  122. "Unsupported device scope\n");
  123. start += scope->length;
  124. }
  125. if (*cnt == 0)
  126. return 0;
  127. *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
  128. if (!*devices)
  129. return -ENOMEM;
  130. start = tmp;
  131. index = 0;
  132. while (start < end) {
  133. scope = start;
  134. if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  135. scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
  136. ret = dmar_parse_one_dev_scope(scope,
  137. &(*devices)[index], segment);
  138. if (ret) {
  139. kfree(*devices);
  140. return ret;
  141. }
  142. index ++;
  143. }
  144. start += scope->length;
  145. }
  146. return 0;
  147. }
  148. /**
  149. * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
  150. * structure which uniquely represent one DMA remapping hardware unit
  151. * present in the platform
  152. */
  153. static int __init
  154. dmar_parse_one_drhd(struct acpi_dmar_header *header)
  155. {
  156. struct acpi_dmar_hardware_unit *drhd;
  157. struct dmar_drhd_unit *dmaru;
  158. int ret = 0;
  159. dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
  160. if (!dmaru)
  161. return -ENOMEM;
  162. dmaru->hdr = header;
  163. drhd = (struct acpi_dmar_hardware_unit *)header;
  164. dmaru->reg_base_addr = drhd->address;
  165. dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
  166. ret = alloc_iommu(dmaru);
  167. if (ret) {
  168. kfree(dmaru);
  169. return ret;
  170. }
  171. dmar_register_drhd_unit(dmaru);
  172. return 0;
  173. }
  174. static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
  175. {
  176. struct acpi_dmar_hardware_unit *drhd;
  177. int ret = 0;
  178. drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
  179. if (dmaru->include_all)
  180. return 0;
  181. ret = dmar_parse_dev_scope((void *)(drhd + 1),
  182. ((void *)drhd) + drhd->header.length,
  183. &dmaru->devices_cnt, &dmaru->devices,
  184. drhd->segment);
  185. if (ret) {
  186. list_del(&dmaru->list);
  187. kfree(dmaru);
  188. }
  189. return ret;
  190. }
  191. #ifdef CONFIG_DMAR
  192. LIST_HEAD(dmar_rmrr_units);
  193. static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
  194. {
  195. list_add(&rmrr->list, &dmar_rmrr_units);
  196. }
  197. static int __init
  198. dmar_parse_one_rmrr(struct acpi_dmar_header *header)
  199. {
  200. struct acpi_dmar_reserved_memory *rmrr;
  201. struct dmar_rmrr_unit *rmrru;
  202. rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
  203. if (!rmrru)
  204. return -ENOMEM;
  205. rmrru->hdr = header;
  206. rmrr = (struct acpi_dmar_reserved_memory *)header;
  207. rmrru->base_address = rmrr->base_address;
  208. rmrru->end_address = rmrr->end_address;
  209. dmar_register_rmrr_unit(rmrru);
  210. return 0;
  211. }
  212. static int __init
  213. rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
  214. {
  215. struct acpi_dmar_reserved_memory *rmrr;
  216. int ret;
  217. rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
  218. ret = dmar_parse_dev_scope((void *)(rmrr + 1),
  219. ((void *)rmrr) + rmrr->header.length,
  220. &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
  221. if (ret || (rmrru->devices_cnt == 0)) {
  222. list_del(&rmrru->list);
  223. kfree(rmrru);
  224. }
  225. return ret;
  226. }
  227. #endif
  228. static void __init
  229. dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
  230. {
  231. struct acpi_dmar_hardware_unit *drhd;
  232. struct acpi_dmar_reserved_memory *rmrr;
  233. switch (header->type) {
  234. case ACPI_DMAR_TYPE_HARDWARE_UNIT:
  235. drhd = (struct acpi_dmar_hardware_unit *)header;
  236. printk (KERN_INFO PREFIX
  237. "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
  238. drhd->flags, (unsigned long long)drhd->address);
  239. break;
  240. case ACPI_DMAR_TYPE_RESERVED_MEMORY:
  241. rmrr = (struct acpi_dmar_reserved_memory *)header;
  242. printk (KERN_INFO PREFIX
  243. "RMRR base: 0x%016Lx end: 0x%016Lx\n",
  244. (unsigned long long)rmrr->base_address,
  245. (unsigned long long)rmrr->end_address);
  246. break;
  247. }
  248. }
  249. /**
  250. * dmar_table_detect - checks to see if the platform supports DMAR devices
  251. */
  252. static int __init dmar_table_detect(void)
  253. {
  254. acpi_status status = AE_OK;
  255. /* if we could find DMAR table, then there are DMAR devices */
  256. status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
  257. (struct acpi_table_header **)&dmar_tbl,
  258. &dmar_tbl_size);
  259. if (ACPI_SUCCESS(status) && !dmar_tbl) {
  260. printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
  261. status = AE_NOT_FOUND;
  262. }
  263. return (ACPI_SUCCESS(status) ? 1 : 0);
  264. }
  265. /**
  266. * parse_dmar_table - parses the DMA reporting table
  267. */
  268. static int __init
  269. parse_dmar_table(void)
  270. {
  271. struct acpi_table_dmar *dmar;
  272. struct acpi_dmar_header *entry_header;
  273. int ret = 0;
  274. /*
  275. * Do it again, earlier dmar_tbl mapping could be mapped with
  276. * fixed map.
  277. */
  278. dmar_table_detect();
  279. dmar = (struct acpi_table_dmar *)dmar_tbl;
  280. if (!dmar)
  281. return -ENODEV;
  282. if (dmar->width < PAGE_SHIFT - 1) {
  283. printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
  284. return -EINVAL;
  285. }
  286. printk (KERN_INFO PREFIX "Host address width %d\n",
  287. dmar->width + 1);
  288. entry_header = (struct acpi_dmar_header *)(dmar + 1);
  289. while (((unsigned long)entry_header) <
  290. (((unsigned long)dmar) + dmar_tbl->length)) {
  291. /* Avoid looping forever on bad ACPI tables */
  292. if (entry_header->length == 0) {
  293. printk(KERN_WARNING PREFIX
  294. "Invalid 0-length structure\n");
  295. ret = -EINVAL;
  296. break;
  297. }
  298. dmar_table_print_dmar_entry(entry_header);
  299. switch (entry_header->type) {
  300. case ACPI_DMAR_TYPE_HARDWARE_UNIT:
  301. ret = dmar_parse_one_drhd(entry_header);
  302. break;
  303. case ACPI_DMAR_TYPE_RESERVED_MEMORY:
  304. #ifdef CONFIG_DMAR
  305. ret = dmar_parse_one_rmrr(entry_header);
  306. #endif
  307. break;
  308. default:
  309. printk(KERN_WARNING PREFIX
  310. "Unknown DMAR structure type\n");
  311. ret = 0; /* for forward compatibility */
  312. break;
  313. }
  314. if (ret)
  315. break;
  316. entry_header = ((void *)entry_header + entry_header->length);
  317. }
  318. return ret;
  319. }
  320. int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
  321. struct pci_dev *dev)
  322. {
  323. int index;
  324. while (dev) {
  325. for (index = 0; index < cnt; index++)
  326. if (dev == devices[index])
  327. return 1;
  328. /* Check our parent */
  329. dev = dev->bus->self;
  330. }
  331. return 0;
  332. }
  333. struct dmar_drhd_unit *
  334. dmar_find_matched_drhd_unit(struct pci_dev *dev)
  335. {
  336. struct dmar_drhd_unit *dmaru = NULL;
  337. struct acpi_dmar_hardware_unit *drhd;
  338. list_for_each_entry(dmaru, &dmar_drhd_units, list) {
  339. drhd = container_of(dmaru->hdr,
  340. struct acpi_dmar_hardware_unit,
  341. header);
  342. if (dmaru->include_all &&
  343. drhd->segment == pci_domain_nr(dev->bus))
  344. return dmaru;
  345. if (dmar_pci_device_match(dmaru->devices,
  346. dmaru->devices_cnt, dev))
  347. return dmaru;
  348. }
  349. return NULL;
  350. }
  351. int __init dmar_dev_scope_init(void)
  352. {
  353. struct dmar_drhd_unit *drhd, *drhd_n;
  354. int ret = -ENODEV;
  355. list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
  356. ret = dmar_parse_dev(drhd);
  357. if (ret)
  358. return ret;
  359. }
  360. #ifdef CONFIG_DMAR
  361. {
  362. struct dmar_rmrr_unit *rmrr, *rmrr_n;
  363. list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
  364. ret = rmrr_parse_dev(rmrr);
  365. if (ret)
  366. return ret;
  367. }
  368. }
  369. #endif
  370. return ret;
  371. }
  372. int __init dmar_table_init(void)
  373. {
  374. static int dmar_table_initialized;
  375. int ret;
  376. if (dmar_table_initialized)
  377. return 0;
  378. dmar_table_initialized = 1;
  379. ret = parse_dmar_table();
  380. if (ret) {
  381. if (ret != -ENODEV)
  382. printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
  383. return ret;
  384. }
  385. if (list_empty(&dmar_drhd_units)) {
  386. printk(KERN_INFO PREFIX "No DMAR devices found\n");
  387. return -ENODEV;
  388. }
  389. #ifdef CONFIG_DMAR
  390. if (list_empty(&dmar_rmrr_units))
  391. printk(KERN_INFO PREFIX "No RMRR found\n");
  392. #endif
  393. #ifdef CONFIG_INTR_REMAP
  394. parse_ioapics_under_ir();
  395. #endif
  396. return 0;
  397. }
  398. void __init detect_intel_iommu(void)
  399. {
  400. int ret;
  401. ret = dmar_table_detect();
  402. {
  403. #ifdef CONFIG_INTR_REMAP
  404. struct acpi_table_dmar *dmar;
  405. /*
  406. * for now we will disable dma-remapping when interrupt
  407. * remapping is enabled.
  408. * When support for queued invalidation for IOTLB invalidation
  409. * is added, we will not need this any more.
  410. */
  411. dmar = (struct acpi_table_dmar *) dmar_tbl;
  412. if (ret && cpu_has_x2apic && dmar->flags & 0x1)
  413. printk(KERN_INFO
  414. "Queued invalidation will be enabled to support "
  415. "x2apic and Intr-remapping.\n");
  416. #endif
  417. #ifdef CONFIG_DMAR
  418. if (ret && !no_iommu && !iommu_detected && !swiotlb &&
  419. !dmar_disabled)
  420. iommu_detected = 1;
  421. #endif
  422. }
  423. early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
  424. dmar_tbl = NULL;
  425. }
  426. int alloc_iommu(struct dmar_drhd_unit *drhd)
  427. {
  428. struct intel_iommu *iommu;
  429. int map_size;
  430. u32 ver;
  431. static int iommu_allocated = 0;
  432. int agaw = 0;
  433. iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
  434. if (!iommu)
  435. return -ENOMEM;
  436. iommu->seq_id = iommu_allocated++;
  437. iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
  438. if (!iommu->reg) {
  439. printk(KERN_ERR "IOMMU: can't map the region\n");
  440. goto error;
  441. }
  442. iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
  443. iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
  444. #ifdef CONFIG_DMAR
  445. agaw = iommu_calculate_agaw(iommu);
  446. if (agaw < 0) {
  447. printk(KERN_ERR
  448. "Cannot get a valid agaw for iommu (seq_id = %d)\n",
  449. iommu->seq_id);
  450. goto error;
  451. }
  452. #endif
  453. iommu->agaw = agaw;
  454. /* the registers might be more than one page */
  455. map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
  456. cap_max_fault_reg_offset(iommu->cap));
  457. map_size = VTD_PAGE_ALIGN(map_size);
  458. if (map_size > VTD_PAGE_SIZE) {
  459. iounmap(iommu->reg);
  460. iommu->reg = ioremap(drhd->reg_base_addr, map_size);
  461. if (!iommu->reg) {
  462. printk(KERN_ERR "IOMMU: can't map the region\n");
  463. goto error;
  464. }
  465. }
  466. ver = readl(iommu->reg + DMAR_VER_REG);
  467. pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
  468. (unsigned long long)drhd->reg_base_addr,
  469. DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
  470. (unsigned long long)iommu->cap,
  471. (unsigned long long)iommu->ecap);
  472. spin_lock_init(&iommu->register_lock);
  473. drhd->iommu = iommu;
  474. return 0;
  475. error:
  476. kfree(iommu);
  477. return -1;
  478. }
  479. void free_iommu(struct intel_iommu *iommu)
  480. {
  481. if (!iommu)
  482. return;
  483. #ifdef CONFIG_DMAR
  484. free_dmar_iommu(iommu);
  485. #endif
  486. if (iommu->reg)
  487. iounmap(iommu->reg);
  488. kfree(iommu);
  489. }
  490. /*
  491. * Reclaim all the submitted descriptors which have completed its work.
  492. */
  493. static inline void reclaim_free_desc(struct q_inval *qi)
  494. {
  495. while (qi->desc_status[qi->free_tail] == QI_DONE) {
  496. qi->desc_status[qi->free_tail] = QI_FREE;
  497. qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
  498. qi->free_cnt++;
  499. }
  500. }
  501. static int qi_check_fault(struct intel_iommu *iommu, int index)
  502. {
  503. u32 fault;
  504. int head;
  505. struct q_inval *qi = iommu->qi;
  506. int wait_index = (index + 1) % QI_LENGTH;
  507. fault = readl(iommu->reg + DMAR_FSTS_REG);
  508. /*
  509. * If IQE happens, the head points to the descriptor associated
  510. * with the error. No new descriptors are fetched until the IQE
  511. * is cleared.
  512. */
  513. if (fault & DMA_FSTS_IQE) {
  514. head = readl(iommu->reg + DMAR_IQH_REG);
  515. if ((head >> 4) == index) {
  516. memcpy(&qi->desc[index], &qi->desc[wait_index],
  517. sizeof(struct qi_desc));
  518. __iommu_flush_cache(iommu, &qi->desc[index],
  519. sizeof(struct qi_desc));
  520. writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
  521. return -EINVAL;
  522. }
  523. }
  524. return 0;
  525. }
  526. /*
  527. * Submit the queued invalidation descriptor to the remapping
  528. * hardware unit and wait for its completion.
  529. */
  530. int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
  531. {
  532. int rc = 0;
  533. struct q_inval *qi = iommu->qi;
  534. struct qi_desc *hw, wait_desc;
  535. int wait_index, index;
  536. unsigned long flags;
  537. if (!qi)
  538. return 0;
  539. hw = qi->desc;
  540. spin_lock_irqsave(&qi->q_lock, flags);
  541. while (qi->free_cnt < 3) {
  542. spin_unlock_irqrestore(&qi->q_lock, flags);
  543. cpu_relax();
  544. spin_lock_irqsave(&qi->q_lock, flags);
  545. }
  546. index = qi->free_head;
  547. wait_index = (index + 1) % QI_LENGTH;
  548. qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
  549. hw[index] = *desc;
  550. wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
  551. QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
  552. wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
  553. hw[wait_index] = wait_desc;
  554. __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
  555. __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
  556. qi->free_head = (qi->free_head + 2) % QI_LENGTH;
  557. qi->free_cnt -= 2;
  558. /*
  559. * update the HW tail register indicating the presence of
  560. * new descriptors.
  561. */
  562. writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
  563. while (qi->desc_status[wait_index] != QI_DONE) {
  564. /*
  565. * We will leave the interrupts disabled, to prevent interrupt
  566. * context to queue another cmd while a cmd is already submitted
  567. * and waiting for completion on this cpu. This is to avoid
  568. * a deadlock where the interrupt context can wait indefinitely
  569. * for free slots in the queue.
  570. */
  571. rc = qi_check_fault(iommu, index);
  572. if (rc)
  573. goto out;
  574. spin_unlock(&qi->q_lock);
  575. cpu_relax();
  576. spin_lock(&qi->q_lock);
  577. }
  578. out:
  579. qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
  580. reclaim_free_desc(qi);
  581. spin_unlock_irqrestore(&qi->q_lock, flags);
  582. return rc;
  583. }
  584. /*
  585. * Flush the global interrupt entry cache.
  586. */
  587. void qi_global_iec(struct intel_iommu *iommu)
  588. {
  589. struct qi_desc desc;
  590. desc.low = QI_IEC_TYPE;
  591. desc.high = 0;
  592. /* should never fail */
  593. qi_submit_sync(&desc, iommu);
  594. }
  595. int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
  596. u64 type, int non_present_entry_flush)
  597. {
  598. struct qi_desc desc;
  599. if (non_present_entry_flush) {
  600. if (!cap_caching_mode(iommu->cap))
  601. return 1;
  602. else
  603. did = 0;
  604. }
  605. desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
  606. | QI_CC_GRAN(type) | QI_CC_TYPE;
  607. desc.high = 0;
  608. return qi_submit_sync(&desc, iommu);
  609. }
  610. int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
  611. unsigned int size_order, u64 type,
  612. int non_present_entry_flush)
  613. {
  614. u8 dw = 0, dr = 0;
  615. struct qi_desc desc;
  616. int ih = 0;
  617. if (non_present_entry_flush) {
  618. if (!cap_caching_mode(iommu->cap))
  619. return 1;
  620. else
  621. did = 0;
  622. }
  623. if (cap_write_drain(iommu->cap))
  624. dw = 1;
  625. if (cap_read_drain(iommu->cap))
  626. dr = 1;
  627. desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
  628. | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
  629. desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
  630. | QI_IOTLB_AM(size_order);
  631. return qi_submit_sync(&desc, iommu);
  632. }
  633. /*
  634. * Enable Queued Invalidation interface. This is a must to support
  635. * interrupt-remapping. Also used by DMA-remapping, which replaces
  636. * register based IOTLB invalidation.
  637. */
  638. int dmar_enable_qi(struct intel_iommu *iommu)
  639. {
  640. u32 cmd, sts;
  641. unsigned long flags;
  642. struct q_inval *qi;
  643. if (!ecap_qis(iommu->ecap))
  644. return -ENOENT;
  645. /*
  646. * queued invalidation is already setup and enabled.
  647. */
  648. if (iommu->qi)
  649. return 0;
  650. iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
  651. if (!iommu->qi)
  652. return -ENOMEM;
  653. qi = iommu->qi;
  654. qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
  655. if (!qi->desc) {
  656. kfree(qi);
  657. iommu->qi = 0;
  658. return -ENOMEM;
  659. }
  660. qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
  661. if (!qi->desc_status) {
  662. free_page((unsigned long) qi->desc);
  663. kfree(qi);
  664. iommu->qi = 0;
  665. return -ENOMEM;
  666. }
  667. qi->free_head = qi->free_tail = 0;
  668. qi->free_cnt = QI_LENGTH;
  669. spin_lock_init(&qi->q_lock);
  670. spin_lock_irqsave(&iommu->register_lock, flags);
  671. /* write zero to the tail reg */
  672. writel(0, iommu->reg + DMAR_IQT_REG);
  673. dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
  674. cmd = iommu->gcmd | DMA_GCMD_QIE;
  675. iommu->gcmd |= DMA_GCMD_QIE;
  676. writel(cmd, iommu->reg + DMAR_GCMD_REG);
  677. /* Make sure hardware complete it */
  678. IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
  679. spin_unlock_irqrestore(&iommu->register_lock, flags);
  680. return 0;
  681. }