amd_iommu.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453
  1. /*
  2. * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
  3. * Author: Joerg Roedel <joerg.roedel@amd.com>
  4. * Leo Duran <leo.duran@amd.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/pci.h>
  20. #include <linux/gfp.h>
  21. #include <linux/bitops.h>
  22. #include <linux/scatterlist.h>
  23. #include <linux/iommu-helper.h>
  24. #include <asm/proto.h>
  25. #include <asm/iommu.h>
  26. #include <asm/gart.h>
  27. #include <asm/amd_iommu_types.h>
  28. #include <asm/amd_iommu.h>
  29. #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  30. #define EXIT_LOOP_COUNT 10000000
  31. static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  32. /* A list of preallocated protection domains */
  33. static LIST_HEAD(iommu_pd_list);
  34. static DEFINE_SPINLOCK(iommu_pd_list_lock);
  35. /*
  36. * general struct to manage commands send to an IOMMU
  37. */
  38. struct iommu_cmd {
  39. u32 data[4];
  40. };
  41. static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  42. struct unity_map_entry *e);
  43. /* returns !0 if the IOMMU is caching non-present entries in its TLB */
  44. static int iommu_has_npcache(struct amd_iommu *iommu)
  45. {
  46. return iommu->cap & (1UL << IOMMU_CAP_NPCACHE);
  47. }
  48. /****************************************************************************
  49. *
  50. * Interrupt handling functions
  51. *
  52. ****************************************************************************/
  53. static void iommu_print_event(void *__evt)
  54. {
  55. u32 *event = __evt;
  56. int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
  57. int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
  58. int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
  59. int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
  60. u64 address = (u64)(((u64)event[3]) << 32) | event[2];
  61. printk(KERN_ERR "AMD IOMMU: Event logged [");
  62. switch (type) {
  63. case EVENT_TYPE_ILL_DEV:
  64. printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
  65. "address=0x%016llx flags=0x%04x]\n",
  66. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  67. address, flags);
  68. break;
  69. case EVENT_TYPE_IO_FAULT:
  70. printk("IO_PAGE_FAULT device=%02x:%02x.%x "
  71. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  72. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  73. domid, address, flags);
  74. break;
  75. case EVENT_TYPE_DEV_TAB_ERR:
  76. printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  77. "address=0x%016llx flags=0x%04x]\n",
  78. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  79. address, flags);
  80. break;
  81. case EVENT_TYPE_PAGE_TAB_ERR:
  82. printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  83. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  84. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  85. domid, address, flags);
  86. break;
  87. case EVENT_TYPE_ILL_CMD:
  88. printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
  89. break;
  90. case EVENT_TYPE_CMD_HARD_ERR:
  91. printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
  92. "flags=0x%04x]\n", address, flags);
  93. break;
  94. case EVENT_TYPE_IOTLB_INV_TO:
  95. printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
  96. "address=0x%016llx]\n",
  97. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  98. address);
  99. break;
  100. case EVENT_TYPE_INV_DEV_REQ:
  101. printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
  102. "address=0x%016llx flags=0x%04x]\n",
  103. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  104. address, flags);
  105. break;
  106. default:
  107. printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
  108. }
  109. }
  110. static void iommu_poll_events(struct amd_iommu *iommu)
  111. {
  112. u32 head, tail;
  113. unsigned long flags;
  114. spin_lock_irqsave(&iommu->lock, flags);
  115. head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  116. tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
  117. while (head != tail) {
  118. iommu_print_event(iommu->evt_buf + head);
  119. head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
  120. }
  121. writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  122. spin_unlock_irqrestore(&iommu->lock, flags);
  123. }
  124. irqreturn_t amd_iommu_int_handler(int irq, void *data)
  125. {
  126. struct amd_iommu *iommu;
  127. list_for_each_entry(iommu, &amd_iommu_list, list)
  128. iommu_poll_events(iommu);
  129. return IRQ_HANDLED;
  130. }
  131. /****************************************************************************
  132. *
  133. * IOMMU command queuing functions
  134. *
  135. ****************************************************************************/
  136. /*
  137. * Writes the command to the IOMMUs command buffer and informs the
  138. * hardware about the new command. Must be called with iommu->lock held.
  139. */
  140. static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  141. {
  142. u32 tail, head;
  143. u8 *target;
  144. tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  145. target = iommu->cmd_buf + tail;
  146. memcpy_toio(target, cmd, sizeof(*cmd));
  147. tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  148. head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  149. if (tail == head)
  150. return -ENOMEM;
  151. writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  152. return 0;
  153. }
  154. /*
  155. * General queuing function for commands. Takes iommu->lock and calls
  156. * __iommu_queue_command().
  157. */
  158. static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  159. {
  160. unsigned long flags;
  161. int ret;
  162. spin_lock_irqsave(&iommu->lock, flags);
  163. ret = __iommu_queue_command(iommu, cmd);
  164. if (!ret)
  165. iommu->need_sync = 1;
  166. spin_unlock_irqrestore(&iommu->lock, flags);
  167. return ret;
  168. }
  169. /*
  170. * This function waits until an IOMMU has completed a completion
  171. * wait command
  172. */
  173. static void __iommu_wait_for_completion(struct amd_iommu *iommu)
  174. {
  175. int ready = 0;
  176. unsigned status = 0;
  177. unsigned long i = 0;
  178. while (!ready && (i < EXIT_LOOP_COUNT)) {
  179. ++i;
  180. /* wait for the bit to become one */
  181. status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
  182. ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
  183. }
  184. /* set bit back to zero */
  185. status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
  186. writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
  187. if (unlikely(i == EXIT_LOOP_COUNT))
  188. panic("AMD IOMMU: Completion wait loop failed\n");
  189. }
  190. /*
  191. * This function queues a completion wait command into the command
  192. * buffer of an IOMMU
  193. */
  194. static int __iommu_completion_wait(struct amd_iommu *iommu)
  195. {
  196. struct iommu_cmd cmd;
  197. memset(&cmd, 0, sizeof(cmd));
  198. cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
  199. CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
  200. return __iommu_queue_command(iommu, &cmd);
  201. }
  202. /*
  203. * This function is called whenever we need to ensure that the IOMMU has
  204. * completed execution of all commands we sent. It sends a
  205. * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
  206. * us about that by writing a value to a physical address we pass with
  207. * the command.
  208. */
  209. static int iommu_completion_wait(struct amd_iommu *iommu)
  210. {
  211. int ret = 0;
  212. unsigned long flags;
  213. spin_lock_irqsave(&iommu->lock, flags);
  214. if (!iommu->need_sync)
  215. goto out;
  216. ret = __iommu_completion_wait(iommu);
  217. iommu->need_sync = 0;
  218. if (ret)
  219. goto out;
  220. __iommu_wait_for_completion(iommu);
  221. out:
  222. spin_unlock_irqrestore(&iommu->lock, flags);
  223. return 0;
  224. }
  225. /*
  226. * Command send function for invalidating a device table entry
  227. */
  228. static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
  229. {
  230. struct iommu_cmd cmd;
  231. int ret;
  232. BUG_ON(iommu == NULL);
  233. memset(&cmd, 0, sizeof(cmd));
  234. CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
  235. cmd.data[0] = devid;
  236. ret = iommu_queue_command(iommu, &cmd);
  237. return ret;
  238. }
  239. static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
  240. u16 domid, int pde, int s)
  241. {
  242. memset(cmd, 0, sizeof(*cmd));
  243. address &= PAGE_MASK;
  244. CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  245. cmd->data[1] |= domid;
  246. cmd->data[2] = lower_32_bits(address);
  247. cmd->data[3] = upper_32_bits(address);
  248. if (s) /* size bit - we flush more than one 4kb page */
  249. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  250. if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
  251. cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  252. }
  253. /*
  254. * Generic command send function for invalidaing TLB entries
  255. */
  256. static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
  257. u64 address, u16 domid, int pde, int s)
  258. {
  259. struct iommu_cmd cmd;
  260. int ret;
  261. __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
  262. ret = iommu_queue_command(iommu, &cmd);
  263. return ret;
  264. }
  265. /*
  266. * TLB invalidation function which is called from the mapping functions.
  267. * It invalidates a single PTE if the range to flush is within a single
  268. * page. Otherwise it flushes the whole TLB of the IOMMU.
  269. */
  270. static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
  271. u64 address, size_t size)
  272. {
  273. int s = 0;
  274. unsigned pages = iommu_num_pages(address, size, PAGE_SIZE);
  275. address &= PAGE_MASK;
  276. if (pages > 1) {
  277. /*
  278. * If we have to flush more than one page, flush all
  279. * TLB entries for this domain
  280. */
  281. address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  282. s = 1;
  283. }
  284. iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);
  285. return 0;
  286. }
  287. /* Flush the whole IO/TLB for a given protection domain */
  288. static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
  289. {
  290. u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  291. iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
  292. }
  293. #ifdef CONFIG_IOMMU_API
  294. /*
  295. * This function is used to flush the IO/TLB for a given protection domain
  296. * on every IOMMU in the system
  297. */
  298. static void iommu_flush_domain(u16 domid)
  299. {
  300. unsigned long flags;
  301. struct amd_iommu *iommu;
  302. struct iommu_cmd cmd;
  303. __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
  304. domid, 1, 1);
  305. list_for_each_entry(iommu, &amd_iommu_list, list) {
  306. spin_lock_irqsave(&iommu->lock, flags);
  307. __iommu_queue_command(iommu, &cmd);
  308. __iommu_completion_wait(iommu);
  309. __iommu_wait_for_completion(iommu);
  310. spin_unlock_irqrestore(&iommu->lock, flags);
  311. }
  312. }
  313. #endif
  314. /****************************************************************************
  315. *
  316. * The functions below are used the create the page table mappings for
  317. * unity mapped regions.
  318. *
  319. ****************************************************************************/
  320. /*
  321. * Generic mapping functions. It maps a physical address into a DMA
  322. * address space. It allocates the page table pages if necessary.
  323. * In the future it can be extended to a generic mapping function
  324. * supporting all features of AMD IOMMU page tables like level skipping
  325. * and full 64 bit address spaces.
  326. */
  327. static int iommu_map_page(struct protection_domain *dom,
  328. unsigned long bus_addr,
  329. unsigned long phys_addr,
  330. int prot)
  331. {
  332. u64 __pte, *pte, *page;
  333. bus_addr = PAGE_ALIGN(bus_addr);
  334. phys_addr = PAGE_ALIGN(phys_addr);
  335. /* only support 512GB address spaces for now */
  336. if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
  337. return -EINVAL;
  338. pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
  339. if (!IOMMU_PTE_PRESENT(*pte)) {
  340. page = (u64 *)get_zeroed_page(GFP_KERNEL);
  341. if (!page)
  342. return -ENOMEM;
  343. *pte = IOMMU_L2_PDE(virt_to_phys(page));
  344. }
  345. pte = IOMMU_PTE_PAGE(*pte);
  346. pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
  347. if (!IOMMU_PTE_PRESENT(*pte)) {
  348. page = (u64 *)get_zeroed_page(GFP_KERNEL);
  349. if (!page)
  350. return -ENOMEM;
  351. *pte = IOMMU_L1_PDE(virt_to_phys(page));
  352. }
  353. pte = IOMMU_PTE_PAGE(*pte);
  354. pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
  355. if (IOMMU_PTE_PRESENT(*pte))
  356. return -EBUSY;
  357. __pte = phys_addr | IOMMU_PTE_P;
  358. if (prot & IOMMU_PROT_IR)
  359. __pte |= IOMMU_PTE_IR;
  360. if (prot & IOMMU_PROT_IW)
  361. __pte |= IOMMU_PTE_IW;
  362. *pte = __pte;
  363. return 0;
  364. }
  365. /*
  366. * This function checks if a specific unity mapping entry is needed for
  367. * this specific IOMMU.
  368. */
  369. static int iommu_for_unity_map(struct amd_iommu *iommu,
  370. struct unity_map_entry *entry)
  371. {
  372. u16 bdf, i;
  373. for (i = entry->devid_start; i <= entry->devid_end; ++i) {
  374. bdf = amd_iommu_alias_table[i];
  375. if (amd_iommu_rlookup_table[bdf] == iommu)
  376. return 1;
  377. }
  378. return 0;
  379. }
  380. /*
  381. * Init the unity mappings for a specific IOMMU in the system
  382. *
  383. * Basically iterates over all unity mapping entries and applies them to
  384. * the default domain DMA of that IOMMU if necessary.
  385. */
  386. static int iommu_init_unity_mappings(struct amd_iommu *iommu)
  387. {
  388. struct unity_map_entry *entry;
  389. int ret;
  390. list_for_each_entry(entry, &amd_iommu_unity_map, list) {
  391. if (!iommu_for_unity_map(iommu, entry))
  392. continue;
  393. ret = dma_ops_unity_map(iommu->default_dom, entry);
  394. if (ret)
  395. return ret;
  396. }
  397. return 0;
  398. }
  399. /*
  400. * This function actually applies the mapping to the page table of the
  401. * dma_ops domain.
  402. */
  403. static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  404. struct unity_map_entry *e)
  405. {
  406. u64 addr;
  407. int ret;
  408. for (addr = e->address_start; addr < e->address_end;
  409. addr += PAGE_SIZE) {
  410. ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
  411. if (ret)
  412. return ret;
  413. /*
  414. * if unity mapping is in aperture range mark the page
  415. * as allocated in the aperture
  416. */
  417. if (addr < dma_dom->aperture_size)
  418. __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
  419. }
  420. return 0;
  421. }
  422. /*
  423. * Inits the unity mappings required for a specific device
  424. */
  425. static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  426. u16 devid)
  427. {
  428. struct unity_map_entry *e;
  429. int ret;
  430. list_for_each_entry(e, &amd_iommu_unity_map, list) {
  431. if (!(devid >= e->devid_start && devid <= e->devid_end))
  432. continue;
  433. ret = dma_ops_unity_map(dma_dom, e);
  434. if (ret)
  435. return ret;
  436. }
  437. return 0;
  438. }
  439. /****************************************************************************
  440. *
  441. * The next functions belong to the address allocator for the dma_ops
  442. * interface functions. They work like the allocators in the other IOMMU
  443. * drivers. Its basically a bitmap which marks the allocated pages in
  444. * the aperture. Maybe it could be enhanced in the future to a more
  445. * efficient allocator.
  446. *
  447. ****************************************************************************/
  448. /*
  449. * The address allocator core function.
  450. *
  451. * called with domain->lock held
  452. */
  453. static unsigned long dma_ops_alloc_addresses(struct device *dev,
  454. struct dma_ops_domain *dom,
  455. unsigned int pages,
  456. unsigned long align_mask,
  457. u64 dma_mask)
  458. {
  459. unsigned long limit;
  460. unsigned long address;
  461. unsigned long boundary_size;
  462. boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
  463. PAGE_SIZE) >> PAGE_SHIFT;
  464. limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
  465. dma_mask >> PAGE_SHIFT);
  466. if (dom->next_bit >= limit) {
  467. dom->next_bit = 0;
  468. dom->need_flush = true;
  469. }
  470. address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
  471. 0 , boundary_size, align_mask);
  472. if (address == -1) {
  473. address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
  474. 0, boundary_size, align_mask);
  475. dom->need_flush = true;
  476. }
  477. if (likely(address != -1)) {
  478. dom->next_bit = address + pages;
  479. address <<= PAGE_SHIFT;
  480. } else
  481. address = bad_dma_address;
  482. WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
  483. return address;
  484. }
  485. /*
  486. * The address free function.
  487. *
  488. * called with domain->lock held
  489. */
  490. static void dma_ops_free_addresses(struct dma_ops_domain *dom,
  491. unsigned long address,
  492. unsigned int pages)
  493. {
  494. address >>= PAGE_SHIFT;
  495. iommu_area_free(dom->bitmap, address, pages);
  496. if (address >= dom->next_bit)
  497. dom->need_flush = true;
  498. }
  499. /****************************************************************************
  500. *
  501. * The next functions belong to the domain allocation. A domain is
  502. * allocated for every IOMMU as the default domain. If device isolation
  503. * is enabled, every device get its own domain. The most important thing
  504. * about domains is the page table mapping the DMA address space they
  505. * contain.
  506. *
  507. ****************************************************************************/
  508. static u16 domain_id_alloc(void)
  509. {
  510. unsigned long flags;
  511. int id;
  512. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  513. id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
  514. BUG_ON(id == 0);
  515. if (id > 0 && id < MAX_DOMAIN_ID)
  516. __set_bit(id, amd_iommu_pd_alloc_bitmap);
  517. else
  518. id = 0;
  519. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  520. return id;
  521. }
  522. #ifdef CONFIG_IOMMU_API
  523. static void domain_id_free(int id)
  524. {
  525. unsigned long flags;
  526. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  527. if (id > 0 && id < MAX_DOMAIN_ID)
  528. __clear_bit(id, amd_iommu_pd_alloc_bitmap);
  529. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  530. }
  531. #endif
  532. /*
  533. * Used to reserve address ranges in the aperture (e.g. for exclusion
  534. * ranges.
  535. */
  536. static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
  537. unsigned long start_page,
  538. unsigned int pages)
  539. {
  540. unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
  541. if (start_page + pages > last_page)
  542. pages = last_page - start_page;
  543. iommu_area_reserve(dom->bitmap, start_page, pages);
  544. }
  545. static void free_pagetable(struct protection_domain *domain)
  546. {
  547. int i, j;
  548. u64 *p1, *p2, *p3;
  549. p1 = domain->pt_root;
  550. if (!p1)
  551. return;
  552. for (i = 0; i < 512; ++i) {
  553. if (!IOMMU_PTE_PRESENT(p1[i]))
  554. continue;
  555. p2 = IOMMU_PTE_PAGE(p1[i]);
  556. for (j = 0; j < 512; ++j) {
  557. if (!IOMMU_PTE_PRESENT(p2[j]))
  558. continue;
  559. p3 = IOMMU_PTE_PAGE(p2[j]);
  560. free_page((unsigned long)p3);
  561. }
  562. free_page((unsigned long)p2);
  563. }
  564. free_page((unsigned long)p1);
  565. domain->pt_root = NULL;
  566. }
  567. /*
  568. * Free a domain, only used if something went wrong in the
  569. * allocation path and we need to free an already allocated page table
  570. */
  571. static void dma_ops_domain_free(struct dma_ops_domain *dom)
  572. {
  573. if (!dom)
  574. return;
  575. free_pagetable(&dom->domain);
  576. kfree(dom->pte_pages);
  577. kfree(dom->bitmap);
  578. kfree(dom);
  579. }
  580. /*
  581. * Allocates a new protection domain usable for the dma_ops functions.
  582. * It also intializes the page table and the address allocator data
  583. * structures required for the dma_ops interface
  584. */
  585. static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
  586. unsigned order)
  587. {
  588. struct dma_ops_domain *dma_dom;
  589. unsigned i, num_pte_pages;
  590. u64 *l2_pde;
  591. u64 address;
  592. /*
  593. * Currently the DMA aperture must be between 32 MB and 1GB in size
  594. */
  595. if ((order < 25) || (order > 30))
  596. return NULL;
  597. dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
  598. if (!dma_dom)
  599. return NULL;
  600. spin_lock_init(&dma_dom->domain.lock);
  601. dma_dom->domain.id = domain_id_alloc();
  602. if (dma_dom->domain.id == 0)
  603. goto free_dma_dom;
  604. dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
  605. dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
  606. dma_dom->domain.priv = dma_dom;
  607. if (!dma_dom->domain.pt_root)
  608. goto free_dma_dom;
  609. dma_dom->aperture_size = (1ULL << order);
  610. dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
  611. GFP_KERNEL);
  612. if (!dma_dom->bitmap)
  613. goto free_dma_dom;
  614. /*
  615. * mark the first page as allocated so we never return 0 as
  616. * a valid dma-address. So we can use 0 as error value
  617. */
  618. dma_dom->bitmap[0] = 1;
  619. dma_dom->next_bit = 0;
  620. dma_dom->need_flush = false;
  621. dma_dom->target_dev = 0xffff;
  622. /* Intialize the exclusion range if necessary */
  623. if (iommu->exclusion_start &&
  624. iommu->exclusion_start < dma_dom->aperture_size) {
  625. unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
  626. int pages = iommu_num_pages(iommu->exclusion_start,
  627. iommu->exclusion_length,
  628. PAGE_SIZE);
  629. dma_ops_reserve_addresses(dma_dom, startpage, pages);
  630. }
  631. /*
  632. * At the last step, build the page tables so we don't need to
  633. * allocate page table pages in the dma_ops mapping/unmapping
  634. * path.
  635. */
  636. num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
  637. dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
  638. GFP_KERNEL);
  639. if (!dma_dom->pte_pages)
  640. goto free_dma_dom;
  641. l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
  642. if (l2_pde == NULL)
  643. goto free_dma_dom;
  644. dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
  645. for (i = 0; i < num_pte_pages; ++i) {
  646. dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
  647. if (!dma_dom->pte_pages[i])
  648. goto free_dma_dom;
  649. address = virt_to_phys(dma_dom->pte_pages[i]);
  650. l2_pde[i] = IOMMU_L1_PDE(address);
  651. }
  652. return dma_dom;
  653. free_dma_dom:
  654. dma_ops_domain_free(dma_dom);
  655. return NULL;
  656. }
  657. /*
  658. * Find out the protection domain structure for a given PCI device. This
  659. * will give us the pointer to the page table root for example.
  660. */
  661. static struct protection_domain *domain_for_device(u16 devid)
  662. {
  663. struct protection_domain *dom;
  664. unsigned long flags;
  665. read_lock_irqsave(&amd_iommu_devtable_lock, flags);
  666. dom = amd_iommu_pd_table[devid];
  667. read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  668. return dom;
  669. }
  670. /*
  671. * If a device is not yet associated with a domain, this function does
  672. * assigns it visible for the hardware
  673. */
  674. static void set_device_domain(struct amd_iommu *iommu,
  675. struct protection_domain *domain,
  676. u16 devid)
  677. {
  678. unsigned long flags;
  679. u64 pte_root = virt_to_phys(domain->pt_root);
  680. pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
  681. << DEV_ENTRY_MODE_SHIFT;
  682. pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
  683. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  684. amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
  685. amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
  686. amd_iommu_dev_table[devid].data[2] = domain->id;
  687. amd_iommu_pd_table[devid] = domain;
  688. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  689. iommu_queue_inv_dev_entry(iommu, devid);
  690. }
  691. /*****************************************************************************
  692. *
  693. * The next functions belong to the dma_ops mapping/unmapping code.
  694. *
  695. *****************************************************************************/
  696. /*
  697. * This function checks if the driver got a valid device from the caller to
  698. * avoid dereferencing invalid pointers.
  699. */
  700. static bool check_device(struct device *dev)
  701. {
  702. if (!dev || !dev->dma_mask)
  703. return false;
  704. return true;
  705. }
  706. /*
  707. * In this function the list of preallocated protection domains is traversed to
  708. * find the domain for a specific device
  709. */
  710. static struct dma_ops_domain *find_protection_domain(u16 devid)
  711. {
  712. struct dma_ops_domain *entry, *ret = NULL;
  713. unsigned long flags;
  714. if (list_empty(&iommu_pd_list))
  715. return NULL;
  716. spin_lock_irqsave(&iommu_pd_list_lock, flags);
  717. list_for_each_entry(entry, &iommu_pd_list, list) {
  718. if (entry->target_dev == devid) {
  719. ret = entry;
  720. break;
  721. }
  722. }
  723. spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  724. return ret;
  725. }
  726. /*
  727. * In the dma_ops path we only have the struct device. This function
  728. * finds the corresponding IOMMU, the protection domain and the
  729. * requestor id for a given device.
  730. * If the device is not yet associated with a domain this is also done
  731. * in this function.
  732. */
  733. static int get_device_resources(struct device *dev,
  734. struct amd_iommu **iommu,
  735. struct protection_domain **domain,
  736. u16 *bdf)
  737. {
  738. struct dma_ops_domain *dma_dom;
  739. struct pci_dev *pcidev;
  740. u16 _bdf;
  741. *iommu = NULL;
  742. *domain = NULL;
  743. *bdf = 0xffff;
  744. if (dev->bus != &pci_bus_type)
  745. return 0;
  746. pcidev = to_pci_dev(dev);
  747. _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
  748. /* device not translated by any IOMMU in the system? */
  749. if (_bdf > amd_iommu_last_bdf)
  750. return 0;
  751. *bdf = amd_iommu_alias_table[_bdf];
  752. *iommu = amd_iommu_rlookup_table[*bdf];
  753. if (*iommu == NULL)
  754. return 0;
  755. *domain = domain_for_device(*bdf);
  756. if (*domain == NULL) {
  757. dma_dom = find_protection_domain(*bdf);
  758. if (!dma_dom)
  759. dma_dom = (*iommu)->default_dom;
  760. *domain = &dma_dom->domain;
  761. set_device_domain(*iommu, *domain, *bdf);
  762. printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
  763. "device ", (*domain)->id);
  764. print_devid(_bdf, 1);
  765. }
  766. if (domain_for_device(_bdf) == NULL)
  767. set_device_domain(*iommu, *domain, _bdf);
  768. return 1;
  769. }
  770. /*
  771. * This is the generic map function. It maps one 4kb page at paddr to
  772. * the given address in the DMA address space for the domain.
  773. */
  774. static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
  775. struct dma_ops_domain *dom,
  776. unsigned long address,
  777. phys_addr_t paddr,
  778. int direction)
  779. {
  780. u64 *pte, __pte;
  781. WARN_ON(address > dom->aperture_size);
  782. paddr &= PAGE_MASK;
  783. pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
  784. pte += IOMMU_PTE_L0_INDEX(address);
  785. __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
  786. if (direction == DMA_TO_DEVICE)
  787. __pte |= IOMMU_PTE_IR;
  788. else if (direction == DMA_FROM_DEVICE)
  789. __pte |= IOMMU_PTE_IW;
  790. else if (direction == DMA_BIDIRECTIONAL)
  791. __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
  792. WARN_ON(*pte);
  793. *pte = __pte;
  794. return (dma_addr_t)address;
  795. }
  796. /*
  797. * The generic unmapping function for on page in the DMA address space.
  798. */
  799. static void dma_ops_domain_unmap(struct amd_iommu *iommu,
  800. struct dma_ops_domain *dom,
  801. unsigned long address)
  802. {
  803. u64 *pte;
  804. if (address >= dom->aperture_size)
  805. return;
  806. WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);
  807. pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
  808. pte += IOMMU_PTE_L0_INDEX(address);
  809. WARN_ON(!*pte);
  810. *pte = 0ULL;
  811. }
  812. /*
  813. * This function contains common code for mapping of a physically
  814. * contiguous memory region into DMA address space. It is used by all
  815. * mapping functions provided with this IOMMU driver.
  816. * Must be called with the domain lock held.
  817. */
  818. static dma_addr_t __map_single(struct device *dev,
  819. struct amd_iommu *iommu,
  820. struct dma_ops_domain *dma_dom,
  821. phys_addr_t paddr,
  822. size_t size,
  823. int dir,
  824. bool align,
  825. u64 dma_mask)
  826. {
  827. dma_addr_t offset = paddr & ~PAGE_MASK;
  828. dma_addr_t address, start;
  829. unsigned int pages;
  830. unsigned long align_mask = 0;
  831. int i;
  832. pages = iommu_num_pages(paddr, size, PAGE_SIZE);
  833. paddr &= PAGE_MASK;
  834. if (align)
  835. align_mask = (1UL << get_order(size)) - 1;
  836. address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
  837. dma_mask);
  838. if (unlikely(address == bad_dma_address))
  839. goto out;
  840. start = address;
  841. for (i = 0; i < pages; ++i) {
  842. dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
  843. paddr += PAGE_SIZE;
  844. start += PAGE_SIZE;
  845. }
  846. address += offset;
  847. if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
  848. iommu_flush_tlb(iommu, dma_dom->domain.id);
  849. dma_dom->need_flush = false;
  850. } else if (unlikely(iommu_has_npcache(iommu)))
  851. iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
  852. out:
  853. return address;
  854. }
  855. /*
  856. * Does the reverse of the __map_single function. Must be called with
  857. * the domain lock held too
  858. */
  859. static void __unmap_single(struct amd_iommu *iommu,
  860. struct dma_ops_domain *dma_dom,
  861. dma_addr_t dma_addr,
  862. size_t size,
  863. int dir)
  864. {
  865. dma_addr_t i, start;
  866. unsigned int pages;
  867. if ((dma_addr == bad_dma_address) ||
  868. (dma_addr + size > dma_dom->aperture_size))
  869. return;
  870. pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
  871. dma_addr &= PAGE_MASK;
  872. start = dma_addr;
  873. for (i = 0; i < pages; ++i) {
  874. dma_ops_domain_unmap(iommu, dma_dom, start);
  875. start += PAGE_SIZE;
  876. }
  877. dma_ops_free_addresses(dma_dom, dma_addr, pages);
  878. if (amd_iommu_unmap_flush || dma_dom->need_flush) {
  879. iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
  880. dma_dom->need_flush = false;
  881. }
  882. }
  883. /*
  884. * The exported map_single function for dma_ops.
  885. */
  886. static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
  887. size_t size, int dir)
  888. {
  889. unsigned long flags;
  890. struct amd_iommu *iommu;
  891. struct protection_domain *domain;
  892. u16 devid;
  893. dma_addr_t addr;
  894. u64 dma_mask;
  895. if (!check_device(dev))
  896. return bad_dma_address;
  897. dma_mask = *dev->dma_mask;
  898. get_device_resources(dev, &iommu, &domain, &devid);
  899. if (iommu == NULL || domain == NULL)
  900. /* device not handled by any AMD IOMMU */
  901. return (dma_addr_t)paddr;
  902. spin_lock_irqsave(&domain->lock, flags);
  903. addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
  904. dma_mask);
  905. if (addr == bad_dma_address)
  906. goto out;
  907. iommu_completion_wait(iommu);
  908. out:
  909. spin_unlock_irqrestore(&domain->lock, flags);
  910. return addr;
  911. }
  912. /*
  913. * The exported unmap_single function for dma_ops.
  914. */
  915. static void unmap_single(struct device *dev, dma_addr_t dma_addr,
  916. size_t size, int dir)
  917. {
  918. unsigned long flags;
  919. struct amd_iommu *iommu;
  920. struct protection_domain *domain;
  921. u16 devid;
  922. if (!check_device(dev) ||
  923. !get_device_resources(dev, &iommu, &domain, &devid))
  924. /* device not handled by any AMD IOMMU */
  925. return;
  926. spin_lock_irqsave(&domain->lock, flags);
  927. __unmap_single(iommu, domain->priv, dma_addr, size, dir);
  928. iommu_completion_wait(iommu);
  929. spin_unlock_irqrestore(&domain->lock, flags);
  930. }
  931. /*
  932. * This is a special map_sg function which is used if we should map a
  933. * device which is not handled by an AMD IOMMU in the system.
  934. */
  935. static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
  936. int nelems, int dir)
  937. {
  938. struct scatterlist *s;
  939. int i;
  940. for_each_sg(sglist, s, nelems, i) {
  941. s->dma_address = (dma_addr_t)sg_phys(s);
  942. s->dma_length = s->length;
  943. }
  944. return nelems;
  945. }
  946. /*
  947. * The exported map_sg function for dma_ops (handles scatter-gather
  948. * lists).
  949. */
  950. static int map_sg(struct device *dev, struct scatterlist *sglist,
  951. int nelems, int dir)
  952. {
  953. unsigned long flags;
  954. struct amd_iommu *iommu;
  955. struct protection_domain *domain;
  956. u16 devid;
  957. int i;
  958. struct scatterlist *s;
  959. phys_addr_t paddr;
  960. int mapped_elems = 0;
  961. u64 dma_mask;
  962. if (!check_device(dev))
  963. return 0;
  964. dma_mask = *dev->dma_mask;
  965. get_device_resources(dev, &iommu, &domain, &devid);
  966. if (!iommu || !domain)
  967. return map_sg_no_iommu(dev, sglist, nelems, dir);
  968. spin_lock_irqsave(&domain->lock, flags);
  969. for_each_sg(sglist, s, nelems, i) {
  970. paddr = sg_phys(s);
  971. s->dma_address = __map_single(dev, iommu, domain->priv,
  972. paddr, s->length, dir, false,
  973. dma_mask);
  974. if (s->dma_address) {
  975. s->dma_length = s->length;
  976. mapped_elems++;
  977. } else
  978. goto unmap;
  979. }
  980. iommu_completion_wait(iommu);
  981. out:
  982. spin_unlock_irqrestore(&domain->lock, flags);
  983. return mapped_elems;
  984. unmap:
  985. for_each_sg(sglist, s, mapped_elems, i) {
  986. if (s->dma_address)
  987. __unmap_single(iommu, domain->priv, s->dma_address,
  988. s->dma_length, dir);
  989. s->dma_address = s->dma_length = 0;
  990. }
  991. mapped_elems = 0;
  992. goto out;
  993. }
  994. /*
  995. * The exported map_sg function for dma_ops (handles scatter-gather
  996. * lists).
  997. */
  998. static void unmap_sg(struct device *dev, struct scatterlist *sglist,
  999. int nelems, int dir)
  1000. {
  1001. unsigned long flags;
  1002. struct amd_iommu *iommu;
  1003. struct protection_domain *domain;
  1004. struct scatterlist *s;
  1005. u16 devid;
  1006. int i;
  1007. if (!check_device(dev) ||
  1008. !get_device_resources(dev, &iommu, &domain, &devid))
  1009. return;
  1010. spin_lock_irqsave(&domain->lock, flags);
  1011. for_each_sg(sglist, s, nelems, i) {
  1012. __unmap_single(iommu, domain->priv, s->dma_address,
  1013. s->dma_length, dir);
  1014. s->dma_address = s->dma_length = 0;
  1015. }
  1016. iommu_completion_wait(iommu);
  1017. spin_unlock_irqrestore(&domain->lock, flags);
  1018. }
  1019. /*
  1020. * The exported alloc_coherent function for dma_ops.
  1021. */
  1022. static void *alloc_coherent(struct device *dev, size_t size,
  1023. dma_addr_t *dma_addr, gfp_t flag)
  1024. {
  1025. unsigned long flags;
  1026. void *virt_addr;
  1027. struct amd_iommu *iommu;
  1028. struct protection_domain *domain;
  1029. u16 devid;
  1030. phys_addr_t paddr;
  1031. u64 dma_mask = dev->coherent_dma_mask;
  1032. if (!check_device(dev))
  1033. return NULL;
  1034. if (!get_device_resources(dev, &iommu, &domain, &devid))
  1035. flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
  1036. flag |= __GFP_ZERO;
  1037. virt_addr = (void *)__get_free_pages(flag, get_order(size));
  1038. if (!virt_addr)
  1039. return 0;
  1040. paddr = virt_to_phys(virt_addr);
  1041. if (!iommu || !domain) {
  1042. *dma_addr = (dma_addr_t)paddr;
  1043. return virt_addr;
  1044. }
  1045. if (!dma_mask)
  1046. dma_mask = *dev->dma_mask;
  1047. spin_lock_irqsave(&domain->lock, flags);
  1048. *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
  1049. size, DMA_BIDIRECTIONAL, true, dma_mask);
  1050. if (*dma_addr == bad_dma_address) {
  1051. free_pages((unsigned long)virt_addr, get_order(size));
  1052. virt_addr = NULL;
  1053. goto out;
  1054. }
  1055. iommu_completion_wait(iommu);
  1056. out:
  1057. spin_unlock_irqrestore(&domain->lock, flags);
  1058. return virt_addr;
  1059. }
  1060. /*
  1061. * The exported free_coherent function for dma_ops.
  1062. */
  1063. static void free_coherent(struct device *dev, size_t size,
  1064. void *virt_addr, dma_addr_t dma_addr)
  1065. {
  1066. unsigned long flags;
  1067. struct amd_iommu *iommu;
  1068. struct protection_domain *domain;
  1069. u16 devid;
  1070. if (!check_device(dev))
  1071. return;
  1072. get_device_resources(dev, &iommu, &domain, &devid);
  1073. if (!iommu || !domain)
  1074. goto free_mem;
  1075. spin_lock_irqsave(&domain->lock, flags);
  1076. __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
  1077. iommu_completion_wait(iommu);
  1078. spin_unlock_irqrestore(&domain->lock, flags);
  1079. free_mem:
  1080. free_pages((unsigned long)virt_addr, get_order(size));
  1081. }
  1082. /*
  1083. * This function is called by the DMA layer to find out if we can handle a
  1084. * particular device. It is part of the dma_ops.
  1085. */
  1086. static int amd_iommu_dma_supported(struct device *dev, u64 mask)
  1087. {
  1088. u16 bdf;
  1089. struct pci_dev *pcidev;
  1090. /* No device or no PCI device */
  1091. if (!dev || dev->bus != &pci_bus_type)
  1092. return 0;
  1093. pcidev = to_pci_dev(dev);
  1094. bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
  1095. /* Out of our scope? */
  1096. if (bdf > amd_iommu_last_bdf)
  1097. return 0;
  1098. return 1;
  1099. }
  1100. /*
  1101. * The function for pre-allocating protection domains.
  1102. *
  1103. * If the driver core informs the DMA layer if a driver grabs a device
  1104. * we don't need to preallocate the protection domains anymore.
  1105. * For now we have to.
  1106. */
  1107. void prealloc_protection_domains(void)
  1108. {
  1109. struct pci_dev *dev = NULL;
  1110. struct dma_ops_domain *dma_dom;
  1111. struct amd_iommu *iommu;
  1112. int order = amd_iommu_aperture_order;
  1113. u16 devid;
  1114. while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
  1115. devid = (dev->bus->number << 8) | dev->devfn;
  1116. if (devid > amd_iommu_last_bdf)
  1117. continue;
  1118. devid = amd_iommu_alias_table[devid];
  1119. if (domain_for_device(devid))
  1120. continue;
  1121. iommu = amd_iommu_rlookup_table[devid];
  1122. if (!iommu)
  1123. continue;
  1124. dma_dom = dma_ops_domain_alloc(iommu, order);
  1125. if (!dma_dom)
  1126. continue;
  1127. init_unity_mappings_for_device(dma_dom, devid);
  1128. dma_dom->target_dev = devid;
  1129. list_add_tail(&dma_dom->list, &iommu_pd_list);
  1130. }
  1131. }
  1132. static struct dma_mapping_ops amd_iommu_dma_ops = {
  1133. .alloc_coherent = alloc_coherent,
  1134. .free_coherent = free_coherent,
  1135. .map_single = map_single,
  1136. .unmap_single = unmap_single,
  1137. .map_sg = map_sg,
  1138. .unmap_sg = unmap_sg,
  1139. .dma_supported = amd_iommu_dma_supported,
  1140. };
  1141. /*
  1142. * The function which clues the AMD IOMMU driver into dma_ops.
  1143. */
  1144. int __init amd_iommu_init_dma_ops(void)
  1145. {
  1146. struct amd_iommu *iommu;
  1147. int order = amd_iommu_aperture_order;
  1148. int ret;
  1149. /*
  1150. * first allocate a default protection domain for every IOMMU we
  1151. * found in the system. Devices not assigned to any other
  1152. * protection domain will be assigned to the default one.
  1153. */
  1154. list_for_each_entry(iommu, &amd_iommu_list, list) {
  1155. iommu->default_dom = dma_ops_domain_alloc(iommu, order);
  1156. if (iommu->default_dom == NULL)
  1157. return -ENOMEM;
  1158. ret = iommu_init_unity_mappings(iommu);
  1159. if (ret)
  1160. goto free_domains;
  1161. }
  1162. /*
  1163. * If device isolation is enabled, pre-allocate the protection
  1164. * domains for each device.
  1165. */
  1166. if (amd_iommu_isolate)
  1167. prealloc_protection_domains();
  1168. iommu_detected = 1;
  1169. force_iommu = 1;
  1170. bad_dma_address = 0;
  1171. #ifdef CONFIG_GART_IOMMU
  1172. gart_iommu_aperture_disabled = 1;
  1173. gart_iommu_aperture = 0;
  1174. #endif
  1175. /* Make the driver finally visible to the drivers */
  1176. dma_ops = &amd_iommu_dma_ops;
  1177. return 0;
  1178. free_domains:
  1179. list_for_each_entry(iommu, &amd_iommu_list, list) {
  1180. if (iommu->default_dom)
  1181. dma_ops_domain_free(iommu->default_dom);
  1182. }
  1183. return ret;
  1184. }