amd_iommu.c 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459
  1. /*
  2. * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
  3. * Author: Joerg Roedel <joerg.roedel@amd.com>
  4. * Leo Duran <leo.duran@amd.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/pci.h>
  20. #include <linux/gfp.h>
  21. #include <linux/bitops.h>
  22. #include <linux/debugfs.h>
  23. #include <linux/scatterlist.h>
  24. #include <linux/dma-mapping.h>
  25. #include <linux/iommu-helper.h>
  26. #include <linux/iommu.h>
  27. #include <asm/proto.h>
  28. #include <asm/iommu.h>
  29. #include <asm/gart.h>
  30. #include <asm/amd_iommu_proto.h>
  31. #include <asm/amd_iommu_types.h>
  32. #include <asm/amd_iommu.h>
  33. #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  34. #define EXIT_LOOP_COUNT 10000000
  35. static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  36. /* A list of preallocated protection domains */
  37. static LIST_HEAD(iommu_pd_list);
  38. static DEFINE_SPINLOCK(iommu_pd_list_lock);
  39. /*
  40. * Domain for untranslated devices - only allocated
  41. * if iommu=pt passed on kernel cmd line.
  42. */
  43. static struct protection_domain *pt_domain;
  44. static struct iommu_ops amd_iommu_ops;
  45. /*
  46. * general struct to manage commands send to an IOMMU
  47. */
  48. struct iommu_cmd {
  49. u32 data[4];
  50. };
  51. static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  52. struct unity_map_entry *e);
  53. static u64 *alloc_pte(struct protection_domain *domain,
  54. unsigned long address, int end_lvl,
  55. u64 **pte_page, gfp_t gfp);
  56. static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
  57. unsigned long start_page,
  58. unsigned int pages);
  59. static void reset_iommu_command_buffer(struct amd_iommu *iommu);
  60. static u64 *fetch_pte(struct protection_domain *domain,
  61. unsigned long address, int map_size);
  62. static void update_domain(struct protection_domain *domain);
  63. /****************************************************************************
  64. *
  65. * Helper functions
  66. *
  67. ****************************************************************************/
  68. static inline u16 get_device_id(struct device *dev)
  69. {
  70. struct pci_dev *pdev = to_pci_dev(dev);
  71. return calc_devid(pdev->bus->number, pdev->devfn);
  72. }
  73. /*
  74. * In this function the list of preallocated protection domains is traversed to
  75. * find the domain for a specific device
  76. */
  77. static struct dma_ops_domain *find_protection_domain(u16 devid)
  78. {
  79. struct dma_ops_domain *entry, *ret = NULL;
  80. unsigned long flags;
  81. u16 alias = amd_iommu_alias_table[devid];
  82. if (list_empty(&iommu_pd_list))
  83. return NULL;
  84. spin_lock_irqsave(&iommu_pd_list_lock, flags);
  85. list_for_each_entry(entry, &iommu_pd_list, list) {
  86. if (entry->target_dev == devid ||
  87. entry->target_dev == alias) {
  88. ret = entry;
  89. break;
  90. }
  91. }
  92. spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  93. return ret;
  94. }
  95. /*
  96. * This function checks if the driver got a valid device from the caller to
  97. * avoid dereferencing invalid pointers.
  98. */
  99. static bool check_device(struct device *dev)
  100. {
  101. u16 devid;
  102. if (!dev || !dev->dma_mask)
  103. return false;
  104. /* No device or no PCI device */
  105. if (!dev || dev->bus != &pci_bus_type)
  106. return false;
  107. devid = get_device_id(dev);
  108. /* Out of our scope? */
  109. if (devid > amd_iommu_last_bdf)
  110. return false;
  111. if (amd_iommu_rlookup_table[devid] == NULL)
  112. return false;
  113. return true;
  114. }
  115. #ifdef CONFIG_AMD_IOMMU_STATS
  116. /*
  117. * Initialization code for statistics collection
  118. */
  119. DECLARE_STATS_COUNTER(compl_wait);
  120. DECLARE_STATS_COUNTER(cnt_map_single);
  121. DECLARE_STATS_COUNTER(cnt_unmap_single);
  122. DECLARE_STATS_COUNTER(cnt_map_sg);
  123. DECLARE_STATS_COUNTER(cnt_unmap_sg);
  124. DECLARE_STATS_COUNTER(cnt_alloc_coherent);
  125. DECLARE_STATS_COUNTER(cnt_free_coherent);
  126. DECLARE_STATS_COUNTER(cross_page);
  127. DECLARE_STATS_COUNTER(domain_flush_single);
  128. DECLARE_STATS_COUNTER(domain_flush_all);
  129. DECLARE_STATS_COUNTER(alloced_io_mem);
  130. DECLARE_STATS_COUNTER(total_map_requests);
  131. static struct dentry *stats_dir;
  132. static struct dentry *de_isolate;
  133. static struct dentry *de_fflush;
  134. static void amd_iommu_stats_add(struct __iommu_counter *cnt)
  135. {
  136. if (stats_dir == NULL)
  137. return;
  138. cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
  139. &cnt->value);
  140. }
  141. static void amd_iommu_stats_init(void)
  142. {
  143. stats_dir = debugfs_create_dir("amd-iommu", NULL);
  144. if (stats_dir == NULL)
  145. return;
  146. de_isolate = debugfs_create_bool("isolation", 0444, stats_dir,
  147. (u32 *)&amd_iommu_isolate);
  148. de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
  149. (u32 *)&amd_iommu_unmap_flush);
  150. amd_iommu_stats_add(&compl_wait);
  151. amd_iommu_stats_add(&cnt_map_single);
  152. amd_iommu_stats_add(&cnt_unmap_single);
  153. amd_iommu_stats_add(&cnt_map_sg);
  154. amd_iommu_stats_add(&cnt_unmap_sg);
  155. amd_iommu_stats_add(&cnt_alloc_coherent);
  156. amd_iommu_stats_add(&cnt_free_coherent);
  157. amd_iommu_stats_add(&cross_page);
  158. amd_iommu_stats_add(&domain_flush_single);
  159. amd_iommu_stats_add(&domain_flush_all);
  160. amd_iommu_stats_add(&alloced_io_mem);
  161. amd_iommu_stats_add(&total_map_requests);
  162. }
  163. #endif
  164. /****************************************************************************
  165. *
  166. * Interrupt handling functions
  167. *
  168. ****************************************************************************/
  169. static void dump_dte_entry(u16 devid)
  170. {
  171. int i;
  172. for (i = 0; i < 8; ++i)
  173. pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
  174. amd_iommu_dev_table[devid].data[i]);
  175. }
  176. static void dump_command(unsigned long phys_addr)
  177. {
  178. struct iommu_cmd *cmd = phys_to_virt(phys_addr);
  179. int i;
  180. for (i = 0; i < 4; ++i)
  181. pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
  182. }
  183. static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
  184. {
  185. u32 *event = __evt;
  186. int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
  187. int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
  188. int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
  189. int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
  190. u64 address = (u64)(((u64)event[3]) << 32) | event[2];
  191. printk(KERN_ERR "AMD-Vi: Event logged [");
  192. switch (type) {
  193. case EVENT_TYPE_ILL_DEV:
  194. printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
  195. "address=0x%016llx flags=0x%04x]\n",
  196. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  197. address, flags);
  198. dump_dte_entry(devid);
  199. break;
  200. case EVENT_TYPE_IO_FAULT:
  201. printk("IO_PAGE_FAULT device=%02x:%02x.%x "
  202. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  203. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  204. domid, address, flags);
  205. break;
  206. case EVENT_TYPE_DEV_TAB_ERR:
  207. printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  208. "address=0x%016llx flags=0x%04x]\n",
  209. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  210. address, flags);
  211. break;
  212. case EVENT_TYPE_PAGE_TAB_ERR:
  213. printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  214. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  215. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  216. domid, address, flags);
  217. break;
  218. case EVENT_TYPE_ILL_CMD:
  219. printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
  220. reset_iommu_command_buffer(iommu);
  221. dump_command(address);
  222. break;
  223. case EVENT_TYPE_CMD_HARD_ERR:
  224. printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
  225. "flags=0x%04x]\n", address, flags);
  226. break;
  227. case EVENT_TYPE_IOTLB_INV_TO:
  228. printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
  229. "address=0x%016llx]\n",
  230. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  231. address);
  232. break;
  233. case EVENT_TYPE_INV_DEV_REQ:
  234. printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
  235. "address=0x%016llx flags=0x%04x]\n",
  236. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  237. address, flags);
  238. break;
  239. default:
  240. printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
  241. }
  242. }
  243. static void iommu_poll_events(struct amd_iommu *iommu)
  244. {
  245. u32 head, tail;
  246. unsigned long flags;
  247. spin_lock_irqsave(&iommu->lock, flags);
  248. head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  249. tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
  250. while (head != tail) {
  251. iommu_print_event(iommu, iommu->evt_buf + head);
  252. head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
  253. }
  254. writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  255. spin_unlock_irqrestore(&iommu->lock, flags);
  256. }
  257. irqreturn_t amd_iommu_int_handler(int irq, void *data)
  258. {
  259. struct amd_iommu *iommu;
  260. for_each_iommu(iommu)
  261. iommu_poll_events(iommu);
  262. return IRQ_HANDLED;
  263. }
  264. /****************************************************************************
  265. *
  266. * IOMMU command queuing functions
  267. *
  268. ****************************************************************************/
  269. /*
  270. * Writes the command to the IOMMUs command buffer and informs the
  271. * hardware about the new command. Must be called with iommu->lock held.
  272. */
  273. static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  274. {
  275. u32 tail, head;
  276. u8 *target;
  277. tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  278. target = iommu->cmd_buf + tail;
  279. memcpy_toio(target, cmd, sizeof(*cmd));
  280. tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  281. head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  282. if (tail == head)
  283. return -ENOMEM;
  284. writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  285. return 0;
  286. }
  287. /*
  288. * General queuing function for commands. Takes iommu->lock and calls
  289. * __iommu_queue_command().
  290. */
  291. static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  292. {
  293. unsigned long flags;
  294. int ret;
  295. spin_lock_irqsave(&iommu->lock, flags);
  296. ret = __iommu_queue_command(iommu, cmd);
  297. if (!ret)
  298. iommu->need_sync = true;
  299. spin_unlock_irqrestore(&iommu->lock, flags);
  300. return ret;
  301. }
  302. /*
  303. * This function waits until an IOMMU has completed a completion
  304. * wait command
  305. */
  306. static void __iommu_wait_for_completion(struct amd_iommu *iommu)
  307. {
  308. int ready = 0;
  309. unsigned status = 0;
  310. unsigned long i = 0;
  311. INC_STATS_COUNTER(compl_wait);
  312. while (!ready && (i < EXIT_LOOP_COUNT)) {
  313. ++i;
  314. /* wait for the bit to become one */
  315. status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
  316. ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
  317. }
  318. /* set bit back to zero */
  319. status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
  320. writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
  321. if (unlikely(i == EXIT_LOOP_COUNT)) {
  322. spin_unlock(&iommu->lock);
  323. reset_iommu_command_buffer(iommu);
  324. spin_lock(&iommu->lock);
  325. }
  326. }
  327. /*
  328. * This function queues a completion wait command into the command
  329. * buffer of an IOMMU
  330. */
  331. static int __iommu_completion_wait(struct amd_iommu *iommu)
  332. {
  333. struct iommu_cmd cmd;
  334. memset(&cmd, 0, sizeof(cmd));
  335. cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
  336. CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
  337. return __iommu_queue_command(iommu, &cmd);
  338. }
  339. /*
  340. * This function is called whenever we need to ensure that the IOMMU has
  341. * completed execution of all commands we sent. It sends a
  342. * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
  343. * us about that by writing a value to a physical address we pass with
  344. * the command.
  345. */
  346. static int iommu_completion_wait(struct amd_iommu *iommu)
  347. {
  348. int ret = 0;
  349. unsigned long flags;
  350. spin_lock_irqsave(&iommu->lock, flags);
  351. if (!iommu->need_sync)
  352. goto out;
  353. ret = __iommu_completion_wait(iommu);
  354. iommu->need_sync = false;
  355. if (ret)
  356. goto out;
  357. __iommu_wait_for_completion(iommu);
  358. out:
  359. spin_unlock_irqrestore(&iommu->lock, flags);
  360. return 0;
  361. }
  362. static void iommu_flush_complete(struct protection_domain *domain)
  363. {
  364. int i;
  365. for (i = 0; i < amd_iommus_present; ++i) {
  366. if (!domain->dev_iommu[i])
  367. continue;
  368. /*
  369. * Devices of this domain are behind this IOMMU
  370. * We need to wait for completion of all commands.
  371. */
  372. iommu_completion_wait(amd_iommus[i]);
  373. }
  374. }
  375. /*
  376. * Command send function for invalidating a device table entry
  377. */
  378. static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
  379. {
  380. struct iommu_cmd cmd;
  381. int ret;
  382. BUG_ON(iommu == NULL);
  383. memset(&cmd, 0, sizeof(cmd));
  384. CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
  385. cmd.data[0] = devid;
  386. ret = iommu_queue_command(iommu, &cmd);
  387. return ret;
  388. }
  389. static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
  390. u16 domid, int pde, int s)
  391. {
  392. memset(cmd, 0, sizeof(*cmd));
  393. address &= PAGE_MASK;
  394. CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  395. cmd->data[1] |= domid;
  396. cmd->data[2] = lower_32_bits(address);
  397. cmd->data[3] = upper_32_bits(address);
  398. if (s) /* size bit - we flush more than one 4kb page */
  399. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  400. if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
  401. cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  402. }
  403. /*
  404. * Generic command send function for invalidaing TLB entries
  405. */
  406. static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
  407. u64 address, u16 domid, int pde, int s)
  408. {
  409. struct iommu_cmd cmd;
  410. int ret;
  411. __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
  412. ret = iommu_queue_command(iommu, &cmd);
  413. return ret;
  414. }
  415. /*
  416. * TLB invalidation function which is called from the mapping functions.
  417. * It invalidates a single PTE if the range to flush is within a single
  418. * page. Otherwise it flushes the whole TLB of the IOMMU.
  419. */
  420. static void __iommu_flush_pages(struct protection_domain *domain,
  421. u64 address, size_t size, int pde)
  422. {
  423. int s = 0, i;
  424. unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
  425. address &= PAGE_MASK;
  426. if (pages > 1) {
  427. /*
  428. * If we have to flush more than one page, flush all
  429. * TLB entries for this domain
  430. */
  431. address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  432. s = 1;
  433. }
  434. for (i = 0; i < amd_iommus_present; ++i) {
  435. if (!domain->dev_iommu[i])
  436. continue;
  437. /*
  438. * Devices of this domain are behind this IOMMU
  439. * We need a TLB flush
  440. */
  441. iommu_queue_inv_iommu_pages(amd_iommus[i], address,
  442. domain->id, pde, s);
  443. }
  444. return;
  445. }
  446. static void iommu_flush_pages(struct protection_domain *domain,
  447. u64 address, size_t size)
  448. {
  449. __iommu_flush_pages(domain, address, size, 0);
  450. }
  451. /* Flush the whole IO/TLB for a given protection domain */
  452. static void iommu_flush_tlb(struct protection_domain *domain)
  453. {
  454. __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
  455. }
  456. /* Flush the whole IO/TLB for a given protection domain - including PDE */
  457. static void iommu_flush_tlb_pde(struct protection_domain *domain)
  458. {
  459. __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
  460. }
  461. /*
  462. * This function flushes all domains that have devices on the given IOMMU
  463. */
  464. static void flush_all_domains_on_iommu(struct amd_iommu *iommu)
  465. {
  466. u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  467. struct protection_domain *domain;
  468. unsigned long flags;
  469. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  470. list_for_each_entry(domain, &amd_iommu_pd_list, list) {
  471. if (domain->dev_iommu[iommu->index] == 0)
  472. continue;
  473. spin_lock(&domain->lock);
  474. iommu_queue_inv_iommu_pages(iommu, address, domain->id, 1, 1);
  475. iommu_flush_complete(domain);
  476. spin_unlock(&domain->lock);
  477. }
  478. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  479. }
  480. /*
  481. * This function uses heavy locking and may disable irqs for some time. But
  482. * this is no issue because it is only called during resume.
  483. */
  484. void amd_iommu_flush_all_domains(void)
  485. {
  486. struct protection_domain *domain;
  487. unsigned long flags;
  488. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  489. list_for_each_entry(domain, &amd_iommu_pd_list, list) {
  490. spin_lock(&domain->lock);
  491. iommu_flush_tlb_pde(domain);
  492. iommu_flush_complete(domain);
  493. spin_unlock(&domain->lock);
  494. }
  495. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  496. }
  497. static void flush_all_devices_for_iommu(struct amd_iommu *iommu)
  498. {
  499. int i;
  500. for (i = 0; i <= amd_iommu_last_bdf; ++i) {
  501. if (iommu != amd_iommu_rlookup_table[i])
  502. continue;
  503. iommu_queue_inv_dev_entry(iommu, i);
  504. iommu_completion_wait(iommu);
  505. }
  506. }
  507. static void flush_devices_by_domain(struct protection_domain *domain)
  508. {
  509. struct amd_iommu *iommu;
  510. int i;
  511. for (i = 0; i <= amd_iommu_last_bdf; ++i) {
  512. if ((domain == NULL && amd_iommu_pd_table[i] == NULL) ||
  513. (amd_iommu_pd_table[i] != domain))
  514. continue;
  515. iommu = amd_iommu_rlookup_table[i];
  516. if (!iommu)
  517. continue;
  518. iommu_queue_inv_dev_entry(iommu, i);
  519. iommu_completion_wait(iommu);
  520. }
  521. }
  522. static void reset_iommu_command_buffer(struct amd_iommu *iommu)
  523. {
  524. pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
  525. if (iommu->reset_in_progress)
  526. panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
  527. iommu->reset_in_progress = true;
  528. amd_iommu_reset_cmd_buffer(iommu);
  529. flush_all_devices_for_iommu(iommu);
  530. flush_all_domains_on_iommu(iommu);
  531. iommu->reset_in_progress = false;
  532. }
  533. void amd_iommu_flush_all_devices(void)
  534. {
  535. flush_devices_by_domain(NULL);
  536. }
  537. /****************************************************************************
  538. *
  539. * The functions below are used the create the page table mappings for
  540. * unity mapped regions.
  541. *
  542. ****************************************************************************/
  543. /*
  544. * Generic mapping functions. It maps a physical address into a DMA
  545. * address space. It allocates the page table pages if necessary.
  546. * In the future it can be extended to a generic mapping function
  547. * supporting all features of AMD IOMMU page tables like level skipping
  548. * and full 64 bit address spaces.
  549. */
  550. static int iommu_map_page(struct protection_domain *dom,
  551. unsigned long bus_addr,
  552. unsigned long phys_addr,
  553. int prot,
  554. int map_size)
  555. {
  556. u64 __pte, *pte;
  557. bus_addr = PAGE_ALIGN(bus_addr);
  558. phys_addr = PAGE_ALIGN(phys_addr);
  559. BUG_ON(!PM_ALIGNED(map_size, bus_addr));
  560. BUG_ON(!PM_ALIGNED(map_size, phys_addr));
  561. if (!(prot & IOMMU_PROT_MASK))
  562. return -EINVAL;
  563. pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
  564. if (IOMMU_PTE_PRESENT(*pte))
  565. return -EBUSY;
  566. __pte = phys_addr | IOMMU_PTE_P;
  567. if (prot & IOMMU_PROT_IR)
  568. __pte |= IOMMU_PTE_IR;
  569. if (prot & IOMMU_PROT_IW)
  570. __pte |= IOMMU_PTE_IW;
  571. *pte = __pte;
  572. update_domain(dom);
  573. return 0;
  574. }
  575. static void iommu_unmap_page(struct protection_domain *dom,
  576. unsigned long bus_addr, int map_size)
  577. {
  578. u64 *pte = fetch_pte(dom, bus_addr, map_size);
  579. if (pte)
  580. *pte = 0;
  581. }
  582. /*
  583. * This function checks if a specific unity mapping entry is needed for
  584. * this specific IOMMU.
  585. */
  586. static int iommu_for_unity_map(struct amd_iommu *iommu,
  587. struct unity_map_entry *entry)
  588. {
  589. u16 bdf, i;
  590. for (i = entry->devid_start; i <= entry->devid_end; ++i) {
  591. bdf = amd_iommu_alias_table[i];
  592. if (amd_iommu_rlookup_table[bdf] == iommu)
  593. return 1;
  594. }
  595. return 0;
  596. }
  597. /*
  598. * Init the unity mappings for a specific IOMMU in the system
  599. *
  600. * Basically iterates over all unity mapping entries and applies them to
  601. * the default domain DMA of that IOMMU if necessary.
  602. */
  603. static int iommu_init_unity_mappings(struct amd_iommu *iommu)
  604. {
  605. struct unity_map_entry *entry;
  606. int ret;
  607. list_for_each_entry(entry, &amd_iommu_unity_map, list) {
  608. if (!iommu_for_unity_map(iommu, entry))
  609. continue;
  610. ret = dma_ops_unity_map(iommu->default_dom, entry);
  611. if (ret)
  612. return ret;
  613. }
  614. return 0;
  615. }
  616. /*
  617. * This function actually applies the mapping to the page table of the
  618. * dma_ops domain.
  619. */
  620. static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  621. struct unity_map_entry *e)
  622. {
  623. u64 addr;
  624. int ret;
  625. for (addr = e->address_start; addr < e->address_end;
  626. addr += PAGE_SIZE) {
  627. ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
  628. PM_MAP_4k);
  629. if (ret)
  630. return ret;
  631. /*
  632. * if unity mapping is in aperture range mark the page
  633. * as allocated in the aperture
  634. */
  635. if (addr < dma_dom->aperture_size)
  636. __set_bit(addr >> PAGE_SHIFT,
  637. dma_dom->aperture[0]->bitmap);
  638. }
  639. return 0;
  640. }
  641. /*
  642. * Inits the unity mappings required for a specific device
  643. */
  644. static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  645. u16 devid)
  646. {
  647. struct unity_map_entry *e;
  648. int ret;
  649. list_for_each_entry(e, &amd_iommu_unity_map, list) {
  650. if (!(devid >= e->devid_start && devid <= e->devid_end))
  651. continue;
  652. ret = dma_ops_unity_map(dma_dom, e);
  653. if (ret)
  654. return ret;
  655. }
  656. return 0;
  657. }
  658. /****************************************************************************
  659. *
  660. * The next functions belong to the address allocator for the dma_ops
  661. * interface functions. They work like the allocators in the other IOMMU
  662. * drivers. Its basically a bitmap which marks the allocated pages in
  663. * the aperture. Maybe it could be enhanced in the future to a more
  664. * efficient allocator.
  665. *
  666. ****************************************************************************/
  667. /*
  668. * The address allocator core functions.
  669. *
  670. * called with domain->lock held
  671. */
  672. /*
  673. * This function checks if there is a PTE for a given dma address. If
  674. * there is one, it returns the pointer to it.
  675. */
  676. static u64 *fetch_pte(struct protection_domain *domain,
  677. unsigned long address, int map_size)
  678. {
  679. int level;
  680. u64 *pte;
  681. level = domain->mode - 1;
  682. pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
  683. while (level > map_size) {
  684. if (!IOMMU_PTE_PRESENT(*pte))
  685. return NULL;
  686. level -= 1;
  687. pte = IOMMU_PTE_PAGE(*pte);
  688. pte = &pte[PM_LEVEL_INDEX(level, address)];
  689. if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
  690. pte = NULL;
  691. break;
  692. }
  693. }
  694. return pte;
  695. }
  696. /*
  697. * This function is used to add a new aperture range to an existing
  698. * aperture in case of dma_ops domain allocation or address allocation
  699. * failure.
  700. */
  701. static int alloc_new_range(struct dma_ops_domain *dma_dom,
  702. bool populate, gfp_t gfp)
  703. {
  704. int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
  705. struct amd_iommu *iommu;
  706. int i;
  707. #ifdef CONFIG_IOMMU_STRESS
  708. populate = false;
  709. #endif
  710. if (index >= APERTURE_MAX_RANGES)
  711. return -ENOMEM;
  712. dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
  713. if (!dma_dom->aperture[index])
  714. return -ENOMEM;
  715. dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
  716. if (!dma_dom->aperture[index]->bitmap)
  717. goto out_free;
  718. dma_dom->aperture[index]->offset = dma_dom->aperture_size;
  719. if (populate) {
  720. unsigned long address = dma_dom->aperture_size;
  721. int i, num_ptes = APERTURE_RANGE_PAGES / 512;
  722. u64 *pte, *pte_page;
  723. for (i = 0; i < num_ptes; ++i) {
  724. pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
  725. &pte_page, gfp);
  726. if (!pte)
  727. goto out_free;
  728. dma_dom->aperture[index]->pte_pages[i] = pte_page;
  729. address += APERTURE_RANGE_SIZE / 64;
  730. }
  731. }
  732. dma_dom->aperture_size += APERTURE_RANGE_SIZE;
  733. /* Intialize the exclusion range if necessary */
  734. for_each_iommu(iommu) {
  735. if (iommu->exclusion_start &&
  736. iommu->exclusion_start >= dma_dom->aperture[index]->offset
  737. && iommu->exclusion_start < dma_dom->aperture_size) {
  738. unsigned long startpage;
  739. int pages = iommu_num_pages(iommu->exclusion_start,
  740. iommu->exclusion_length,
  741. PAGE_SIZE);
  742. startpage = iommu->exclusion_start >> PAGE_SHIFT;
  743. dma_ops_reserve_addresses(dma_dom, startpage, pages);
  744. }
  745. }
  746. /*
  747. * Check for areas already mapped as present in the new aperture
  748. * range and mark those pages as reserved in the allocator. Such
  749. * mappings may already exist as a result of requested unity
  750. * mappings for devices.
  751. */
  752. for (i = dma_dom->aperture[index]->offset;
  753. i < dma_dom->aperture_size;
  754. i += PAGE_SIZE) {
  755. u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
  756. if (!pte || !IOMMU_PTE_PRESENT(*pte))
  757. continue;
  758. dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
  759. }
  760. update_domain(&dma_dom->domain);
  761. return 0;
  762. out_free:
  763. update_domain(&dma_dom->domain);
  764. free_page((unsigned long)dma_dom->aperture[index]->bitmap);
  765. kfree(dma_dom->aperture[index]);
  766. dma_dom->aperture[index] = NULL;
  767. return -ENOMEM;
  768. }
  769. static unsigned long dma_ops_area_alloc(struct device *dev,
  770. struct dma_ops_domain *dom,
  771. unsigned int pages,
  772. unsigned long align_mask,
  773. u64 dma_mask,
  774. unsigned long start)
  775. {
  776. unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
  777. int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
  778. int i = start >> APERTURE_RANGE_SHIFT;
  779. unsigned long boundary_size;
  780. unsigned long address = -1;
  781. unsigned long limit;
  782. next_bit >>= PAGE_SHIFT;
  783. boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
  784. PAGE_SIZE) >> PAGE_SHIFT;
  785. for (;i < max_index; ++i) {
  786. unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
  787. if (dom->aperture[i]->offset >= dma_mask)
  788. break;
  789. limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
  790. dma_mask >> PAGE_SHIFT);
  791. address = iommu_area_alloc(dom->aperture[i]->bitmap,
  792. limit, next_bit, pages, 0,
  793. boundary_size, align_mask);
  794. if (address != -1) {
  795. address = dom->aperture[i]->offset +
  796. (address << PAGE_SHIFT);
  797. dom->next_address = address + (pages << PAGE_SHIFT);
  798. break;
  799. }
  800. next_bit = 0;
  801. }
  802. return address;
  803. }
  804. static unsigned long dma_ops_alloc_addresses(struct device *dev,
  805. struct dma_ops_domain *dom,
  806. unsigned int pages,
  807. unsigned long align_mask,
  808. u64 dma_mask)
  809. {
  810. unsigned long address;
  811. #ifdef CONFIG_IOMMU_STRESS
  812. dom->next_address = 0;
  813. dom->need_flush = true;
  814. #endif
  815. address = dma_ops_area_alloc(dev, dom, pages, align_mask,
  816. dma_mask, dom->next_address);
  817. if (address == -1) {
  818. dom->next_address = 0;
  819. address = dma_ops_area_alloc(dev, dom, pages, align_mask,
  820. dma_mask, 0);
  821. dom->need_flush = true;
  822. }
  823. if (unlikely(address == -1))
  824. address = DMA_ERROR_CODE;
  825. WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
  826. return address;
  827. }
  828. /*
  829. * The address free function.
  830. *
  831. * called with domain->lock held
  832. */
  833. static void dma_ops_free_addresses(struct dma_ops_domain *dom,
  834. unsigned long address,
  835. unsigned int pages)
  836. {
  837. unsigned i = address >> APERTURE_RANGE_SHIFT;
  838. struct aperture_range *range = dom->aperture[i];
  839. BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
  840. #ifdef CONFIG_IOMMU_STRESS
  841. if (i < 4)
  842. return;
  843. #endif
  844. if (address >= dom->next_address)
  845. dom->need_flush = true;
  846. address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
  847. iommu_area_free(range->bitmap, address, pages);
  848. }
  849. /****************************************************************************
  850. *
  851. * The next functions belong to the domain allocation. A domain is
  852. * allocated for every IOMMU as the default domain. If device isolation
  853. * is enabled, every device get its own domain. The most important thing
  854. * about domains is the page table mapping the DMA address space they
  855. * contain.
  856. *
  857. ****************************************************************************/
  858. /*
  859. * This function adds a protection domain to the global protection domain list
  860. */
  861. static void add_domain_to_list(struct protection_domain *domain)
  862. {
  863. unsigned long flags;
  864. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  865. list_add(&domain->list, &amd_iommu_pd_list);
  866. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  867. }
  868. /*
  869. * This function removes a protection domain to the global
  870. * protection domain list
  871. */
  872. static void del_domain_from_list(struct protection_domain *domain)
  873. {
  874. unsigned long flags;
  875. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  876. list_del(&domain->list);
  877. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  878. }
  879. static u16 domain_id_alloc(void)
  880. {
  881. unsigned long flags;
  882. int id;
  883. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  884. id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
  885. BUG_ON(id == 0);
  886. if (id > 0 && id < MAX_DOMAIN_ID)
  887. __set_bit(id, amd_iommu_pd_alloc_bitmap);
  888. else
  889. id = 0;
  890. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  891. return id;
  892. }
  893. static void domain_id_free(int id)
  894. {
  895. unsigned long flags;
  896. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  897. if (id > 0 && id < MAX_DOMAIN_ID)
  898. __clear_bit(id, amd_iommu_pd_alloc_bitmap);
  899. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  900. }
  901. /*
  902. * Used to reserve address ranges in the aperture (e.g. for exclusion
  903. * ranges.
  904. */
  905. static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
  906. unsigned long start_page,
  907. unsigned int pages)
  908. {
  909. unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
  910. if (start_page + pages > last_page)
  911. pages = last_page - start_page;
  912. for (i = start_page; i < start_page + pages; ++i) {
  913. int index = i / APERTURE_RANGE_PAGES;
  914. int page = i % APERTURE_RANGE_PAGES;
  915. __set_bit(page, dom->aperture[index]->bitmap);
  916. }
  917. }
  918. static void free_pagetable(struct protection_domain *domain)
  919. {
  920. int i, j;
  921. u64 *p1, *p2, *p3;
  922. p1 = domain->pt_root;
  923. if (!p1)
  924. return;
  925. for (i = 0; i < 512; ++i) {
  926. if (!IOMMU_PTE_PRESENT(p1[i]))
  927. continue;
  928. p2 = IOMMU_PTE_PAGE(p1[i]);
  929. for (j = 0; j < 512; ++j) {
  930. if (!IOMMU_PTE_PRESENT(p2[j]))
  931. continue;
  932. p3 = IOMMU_PTE_PAGE(p2[j]);
  933. free_page((unsigned long)p3);
  934. }
  935. free_page((unsigned long)p2);
  936. }
  937. free_page((unsigned long)p1);
  938. domain->pt_root = NULL;
  939. }
  940. /*
  941. * Free a domain, only used if something went wrong in the
  942. * allocation path and we need to free an already allocated page table
  943. */
  944. static void dma_ops_domain_free(struct dma_ops_domain *dom)
  945. {
  946. int i;
  947. if (!dom)
  948. return;
  949. del_domain_from_list(&dom->domain);
  950. free_pagetable(&dom->domain);
  951. for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
  952. if (!dom->aperture[i])
  953. continue;
  954. free_page((unsigned long)dom->aperture[i]->bitmap);
  955. kfree(dom->aperture[i]);
  956. }
  957. kfree(dom);
  958. }
  959. /*
  960. * Allocates a new protection domain usable for the dma_ops functions.
  961. * It also intializes the page table and the address allocator data
  962. * structures required for the dma_ops interface
  963. */
  964. static struct dma_ops_domain *dma_ops_domain_alloc(void)
  965. {
  966. struct dma_ops_domain *dma_dom;
  967. dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
  968. if (!dma_dom)
  969. return NULL;
  970. spin_lock_init(&dma_dom->domain.lock);
  971. dma_dom->domain.id = domain_id_alloc();
  972. if (dma_dom->domain.id == 0)
  973. goto free_dma_dom;
  974. dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
  975. dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
  976. dma_dom->domain.flags = PD_DMA_OPS_MASK;
  977. dma_dom->domain.priv = dma_dom;
  978. if (!dma_dom->domain.pt_root)
  979. goto free_dma_dom;
  980. dma_dom->need_flush = false;
  981. dma_dom->target_dev = 0xffff;
  982. add_domain_to_list(&dma_dom->domain);
  983. if (alloc_new_range(dma_dom, true, GFP_KERNEL))
  984. goto free_dma_dom;
  985. /*
  986. * mark the first page as allocated so we never return 0 as
  987. * a valid dma-address. So we can use 0 as error value
  988. */
  989. dma_dom->aperture[0]->bitmap[0] = 1;
  990. dma_dom->next_address = 0;
  991. return dma_dom;
  992. free_dma_dom:
  993. dma_ops_domain_free(dma_dom);
  994. return NULL;
  995. }
  996. /*
  997. * little helper function to check whether a given protection domain is a
  998. * dma_ops domain
  999. */
  1000. static bool dma_ops_domain(struct protection_domain *domain)
  1001. {
  1002. return domain->flags & PD_DMA_OPS_MASK;
  1003. }
  1004. static void set_dte_entry(u16 devid, struct protection_domain *domain)
  1005. {
  1006. struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
  1007. u64 pte_root = virt_to_phys(domain->pt_root);
  1008. BUG_ON(amd_iommu_pd_table[devid] != NULL);
  1009. pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
  1010. << DEV_ENTRY_MODE_SHIFT;
  1011. pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
  1012. amd_iommu_dev_table[devid].data[2] = domain->id;
  1013. amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
  1014. amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
  1015. amd_iommu_pd_table[devid] = domain;
  1016. /* Do reference counting */
  1017. domain->dev_iommu[iommu->index] += 1;
  1018. domain->dev_cnt += 1;
  1019. /* Flush the changes DTE entry */
  1020. iommu_queue_inv_dev_entry(iommu, devid);
  1021. }
  1022. static void clear_dte_entry(u16 devid)
  1023. {
  1024. struct protection_domain *domain = amd_iommu_pd_table[devid];
  1025. struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
  1026. BUG_ON(domain == NULL);
  1027. /* remove domain from the lookup table */
  1028. amd_iommu_pd_table[devid] = NULL;
  1029. /* remove entry from the device table seen by the hardware */
  1030. amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
  1031. amd_iommu_dev_table[devid].data[1] = 0;
  1032. amd_iommu_dev_table[devid].data[2] = 0;
  1033. amd_iommu_apply_erratum_63(devid);
  1034. /* decrease reference counters */
  1035. domain->dev_iommu[iommu->index] -= 1;
  1036. domain->dev_cnt -= 1;
  1037. iommu_queue_inv_dev_entry(iommu, devid);
  1038. }
  1039. /*
  1040. * If a device is not yet associated with a domain, this function does
  1041. * assigns it visible for the hardware
  1042. */
  1043. static int __attach_device(struct device *dev,
  1044. struct protection_domain *domain)
  1045. {
  1046. u16 devid = get_device_id(dev);
  1047. u16 alias = amd_iommu_alias_table[devid];
  1048. /* lock domain */
  1049. spin_lock(&domain->lock);
  1050. /* Some sanity checks */
  1051. if (amd_iommu_pd_table[alias] != NULL &&
  1052. amd_iommu_pd_table[alias] != domain)
  1053. return -EBUSY;
  1054. if (amd_iommu_pd_table[devid] != NULL &&
  1055. amd_iommu_pd_table[devid] != domain)
  1056. return -EBUSY;
  1057. /* Do real assignment */
  1058. if (alias != devid &&
  1059. amd_iommu_pd_table[alias] == NULL)
  1060. set_dte_entry(alias, domain);
  1061. if (amd_iommu_pd_table[devid] == NULL)
  1062. set_dte_entry(devid, domain);
  1063. /* ready */
  1064. spin_unlock(&domain->lock);
  1065. return 0;
  1066. }
  1067. /*
  1068. * If a device is not yet associated with a domain, this function does
  1069. * assigns it visible for the hardware
  1070. */
  1071. static int attach_device(struct device *dev,
  1072. struct protection_domain *domain)
  1073. {
  1074. unsigned long flags;
  1075. int ret;
  1076. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1077. ret = __attach_device(dev, domain);
  1078. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1079. /*
  1080. * We might boot into a crash-kernel here. The crashed kernel
  1081. * left the caches in the IOMMU dirty. So we have to flush
  1082. * here to evict all dirty stuff.
  1083. */
  1084. iommu_flush_tlb_pde(domain);
  1085. return ret;
  1086. }
  1087. /*
  1088. * Removes a device from a protection domain (unlocked)
  1089. */
  1090. static void __detach_device(struct device *dev)
  1091. {
  1092. u16 devid = get_device_id(dev);
  1093. struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
  1094. BUG_ON(!iommu);
  1095. clear_dte_entry(devid);
  1096. /*
  1097. * If we run in passthrough mode the device must be assigned to the
  1098. * passthrough domain if it is detached from any other domain
  1099. */
  1100. if (iommu_pass_through)
  1101. __attach_device(dev, pt_domain);
  1102. }
  1103. /*
  1104. * Removes a device from a protection domain (with devtable_lock held)
  1105. */
  1106. static void detach_device(struct device *dev)
  1107. {
  1108. unsigned long flags;
  1109. /* lock device table */
  1110. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1111. __detach_device(dev);
  1112. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1113. }
  1114. /*
  1115. * Find out the protection domain structure for a given PCI device. This
  1116. * will give us the pointer to the page table root for example.
  1117. */
  1118. static struct protection_domain *domain_for_device(struct device *dev)
  1119. {
  1120. struct protection_domain *dom;
  1121. unsigned long flags;
  1122. u16 devid, alias;
  1123. devid = get_device_id(dev);
  1124. alias = amd_iommu_alias_table[devid];
  1125. read_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1126. dom = amd_iommu_pd_table[devid];
  1127. if (dom == NULL &&
  1128. amd_iommu_pd_table[alias] != NULL) {
  1129. __attach_device(dev, amd_iommu_pd_table[alias]);
  1130. dom = amd_iommu_pd_table[devid];
  1131. }
  1132. read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1133. return dom;
  1134. }
  1135. static int device_change_notifier(struct notifier_block *nb,
  1136. unsigned long action, void *data)
  1137. {
  1138. struct device *dev = data;
  1139. u16 devid;
  1140. struct protection_domain *domain;
  1141. struct dma_ops_domain *dma_domain;
  1142. struct amd_iommu *iommu;
  1143. unsigned long flags;
  1144. if (!check_device(dev))
  1145. return 0;
  1146. devid = get_device_id(dev);
  1147. iommu = amd_iommu_rlookup_table[devid];
  1148. domain = domain_for_device(dev);
  1149. if (domain && !dma_ops_domain(domain))
  1150. WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
  1151. "to a non-dma-ops domain\n", dev_name(dev));
  1152. switch (action) {
  1153. case BUS_NOTIFY_UNBOUND_DRIVER:
  1154. if (!domain)
  1155. goto out;
  1156. if (iommu_pass_through)
  1157. break;
  1158. detach_device(dev);
  1159. break;
  1160. case BUS_NOTIFY_ADD_DEVICE:
  1161. /* allocate a protection domain if a device is added */
  1162. dma_domain = find_protection_domain(devid);
  1163. if (dma_domain)
  1164. goto out;
  1165. dma_domain = dma_ops_domain_alloc();
  1166. if (!dma_domain)
  1167. goto out;
  1168. dma_domain->target_dev = devid;
  1169. spin_lock_irqsave(&iommu_pd_list_lock, flags);
  1170. list_add_tail(&dma_domain->list, &iommu_pd_list);
  1171. spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  1172. break;
  1173. default:
  1174. goto out;
  1175. }
  1176. iommu_queue_inv_dev_entry(iommu, devid);
  1177. iommu_completion_wait(iommu);
  1178. out:
  1179. return 0;
  1180. }
  1181. static struct notifier_block device_nb = {
  1182. .notifier_call = device_change_notifier,
  1183. };
  1184. /*****************************************************************************
  1185. *
  1186. * The next functions belong to the dma_ops mapping/unmapping code.
  1187. *
  1188. *****************************************************************************/
  1189. /*
  1190. * In the dma_ops path we only have the struct device. This function
  1191. * finds the corresponding IOMMU, the protection domain and the
  1192. * requestor id for a given device.
  1193. * If the device is not yet associated with a domain this is also done
  1194. * in this function.
  1195. */
  1196. static struct protection_domain *get_domain(struct device *dev)
  1197. {
  1198. struct protection_domain *domain;
  1199. struct dma_ops_domain *dma_dom;
  1200. u16 devid = get_device_id(dev);
  1201. if (!check_device(dev))
  1202. return ERR_PTR(-EINVAL);
  1203. domain = domain_for_device(dev);
  1204. if (domain != NULL && !dma_ops_domain(domain))
  1205. return ERR_PTR(-EBUSY);
  1206. if (domain != NULL)
  1207. return domain;
  1208. /* Device not bount yet - bind it */
  1209. dma_dom = find_protection_domain(devid);
  1210. if (!dma_dom)
  1211. dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
  1212. attach_device(dev, &dma_dom->domain);
  1213. DUMP_printk("Using protection domain %d for device %s\n",
  1214. dma_dom->domain.id, dev_name(dev));
  1215. return &dma_dom->domain;
  1216. }
  1217. static void update_device_table(struct protection_domain *domain)
  1218. {
  1219. unsigned long flags;
  1220. int i;
  1221. for (i = 0; i <= amd_iommu_last_bdf; ++i) {
  1222. if (amd_iommu_pd_table[i] != domain)
  1223. continue;
  1224. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1225. set_dte_entry(i, domain);
  1226. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1227. }
  1228. }
  1229. static void update_domain(struct protection_domain *domain)
  1230. {
  1231. if (!domain->updated)
  1232. return;
  1233. update_device_table(domain);
  1234. flush_devices_by_domain(domain);
  1235. iommu_flush_tlb_pde(domain);
  1236. domain->updated = false;
  1237. }
  1238. /*
  1239. * This function is used to add another level to an IO page table. Adding
  1240. * another level increases the size of the address space by 9 bits to a size up
  1241. * to 64 bits.
  1242. */
  1243. static bool increase_address_space(struct protection_domain *domain,
  1244. gfp_t gfp)
  1245. {
  1246. u64 *pte;
  1247. if (domain->mode == PAGE_MODE_6_LEVEL)
  1248. /* address space already 64 bit large */
  1249. return false;
  1250. pte = (void *)get_zeroed_page(gfp);
  1251. if (!pte)
  1252. return false;
  1253. *pte = PM_LEVEL_PDE(domain->mode,
  1254. virt_to_phys(domain->pt_root));
  1255. domain->pt_root = pte;
  1256. domain->mode += 1;
  1257. domain->updated = true;
  1258. return true;
  1259. }
  1260. static u64 *alloc_pte(struct protection_domain *domain,
  1261. unsigned long address,
  1262. int end_lvl,
  1263. u64 **pte_page,
  1264. gfp_t gfp)
  1265. {
  1266. u64 *pte, *page;
  1267. int level;
  1268. while (address > PM_LEVEL_SIZE(domain->mode))
  1269. increase_address_space(domain, gfp);
  1270. level = domain->mode - 1;
  1271. pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
  1272. while (level > end_lvl) {
  1273. if (!IOMMU_PTE_PRESENT(*pte)) {
  1274. page = (u64 *)get_zeroed_page(gfp);
  1275. if (!page)
  1276. return NULL;
  1277. *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
  1278. }
  1279. level -= 1;
  1280. pte = IOMMU_PTE_PAGE(*pte);
  1281. if (pte_page && level == end_lvl)
  1282. *pte_page = pte;
  1283. pte = &pte[PM_LEVEL_INDEX(level, address)];
  1284. }
  1285. return pte;
  1286. }
  1287. /*
  1288. * This function fetches the PTE for a given address in the aperture
  1289. */
  1290. static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
  1291. unsigned long address)
  1292. {
  1293. struct aperture_range *aperture;
  1294. u64 *pte, *pte_page;
  1295. aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
  1296. if (!aperture)
  1297. return NULL;
  1298. pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
  1299. if (!pte) {
  1300. pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
  1301. GFP_ATOMIC);
  1302. aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
  1303. } else
  1304. pte += PM_LEVEL_INDEX(0, address);
  1305. update_domain(&dom->domain);
  1306. return pte;
  1307. }
  1308. /*
  1309. * This is the generic map function. It maps one 4kb page at paddr to
  1310. * the given address in the DMA address space for the domain.
  1311. */
  1312. static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
  1313. unsigned long address,
  1314. phys_addr_t paddr,
  1315. int direction)
  1316. {
  1317. u64 *pte, __pte;
  1318. WARN_ON(address > dom->aperture_size);
  1319. paddr &= PAGE_MASK;
  1320. pte = dma_ops_get_pte(dom, address);
  1321. if (!pte)
  1322. return DMA_ERROR_CODE;
  1323. __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
  1324. if (direction == DMA_TO_DEVICE)
  1325. __pte |= IOMMU_PTE_IR;
  1326. else if (direction == DMA_FROM_DEVICE)
  1327. __pte |= IOMMU_PTE_IW;
  1328. else if (direction == DMA_BIDIRECTIONAL)
  1329. __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
  1330. WARN_ON(*pte);
  1331. *pte = __pte;
  1332. return (dma_addr_t)address;
  1333. }
  1334. /*
  1335. * The generic unmapping function for on page in the DMA address space.
  1336. */
  1337. static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
  1338. unsigned long address)
  1339. {
  1340. struct aperture_range *aperture;
  1341. u64 *pte;
  1342. if (address >= dom->aperture_size)
  1343. return;
  1344. aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
  1345. if (!aperture)
  1346. return;
  1347. pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
  1348. if (!pte)
  1349. return;
  1350. pte += PM_LEVEL_INDEX(0, address);
  1351. WARN_ON(!*pte);
  1352. *pte = 0ULL;
  1353. }
  1354. /*
  1355. * This function contains common code for mapping of a physically
  1356. * contiguous memory region into DMA address space. It is used by all
  1357. * mapping functions provided with this IOMMU driver.
  1358. * Must be called with the domain lock held.
  1359. */
  1360. static dma_addr_t __map_single(struct device *dev,
  1361. struct dma_ops_domain *dma_dom,
  1362. phys_addr_t paddr,
  1363. size_t size,
  1364. int dir,
  1365. bool align,
  1366. u64 dma_mask)
  1367. {
  1368. dma_addr_t offset = paddr & ~PAGE_MASK;
  1369. dma_addr_t address, start, ret;
  1370. unsigned int pages;
  1371. unsigned long align_mask = 0;
  1372. int i;
  1373. pages = iommu_num_pages(paddr, size, PAGE_SIZE);
  1374. paddr &= PAGE_MASK;
  1375. INC_STATS_COUNTER(total_map_requests);
  1376. if (pages > 1)
  1377. INC_STATS_COUNTER(cross_page);
  1378. if (align)
  1379. align_mask = (1UL << get_order(size)) - 1;
  1380. retry:
  1381. address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
  1382. dma_mask);
  1383. if (unlikely(address == DMA_ERROR_CODE)) {
  1384. /*
  1385. * setting next_address here will let the address
  1386. * allocator only scan the new allocated range in the
  1387. * first run. This is a small optimization.
  1388. */
  1389. dma_dom->next_address = dma_dom->aperture_size;
  1390. if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
  1391. goto out;
  1392. /*
  1393. * aperture was sucessfully enlarged by 128 MB, try
  1394. * allocation again
  1395. */
  1396. goto retry;
  1397. }
  1398. start = address;
  1399. for (i = 0; i < pages; ++i) {
  1400. ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
  1401. if (ret == DMA_ERROR_CODE)
  1402. goto out_unmap;
  1403. paddr += PAGE_SIZE;
  1404. start += PAGE_SIZE;
  1405. }
  1406. address += offset;
  1407. ADD_STATS_COUNTER(alloced_io_mem, size);
  1408. if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
  1409. iommu_flush_tlb(&dma_dom->domain);
  1410. dma_dom->need_flush = false;
  1411. } else if (unlikely(amd_iommu_np_cache))
  1412. iommu_flush_pages(&dma_dom->domain, address, size);
  1413. out:
  1414. return address;
  1415. out_unmap:
  1416. for (--i; i >= 0; --i) {
  1417. start -= PAGE_SIZE;
  1418. dma_ops_domain_unmap(dma_dom, start);
  1419. }
  1420. dma_ops_free_addresses(dma_dom, address, pages);
  1421. return DMA_ERROR_CODE;
  1422. }
  1423. /*
  1424. * Does the reverse of the __map_single function. Must be called with
  1425. * the domain lock held too
  1426. */
  1427. static void __unmap_single(struct dma_ops_domain *dma_dom,
  1428. dma_addr_t dma_addr,
  1429. size_t size,
  1430. int dir)
  1431. {
  1432. dma_addr_t i, start;
  1433. unsigned int pages;
  1434. if ((dma_addr == DMA_ERROR_CODE) ||
  1435. (dma_addr + size > dma_dom->aperture_size))
  1436. return;
  1437. pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
  1438. dma_addr &= PAGE_MASK;
  1439. start = dma_addr;
  1440. for (i = 0; i < pages; ++i) {
  1441. dma_ops_domain_unmap(dma_dom, start);
  1442. start += PAGE_SIZE;
  1443. }
  1444. SUB_STATS_COUNTER(alloced_io_mem, size);
  1445. dma_ops_free_addresses(dma_dom, dma_addr, pages);
  1446. if (amd_iommu_unmap_flush || dma_dom->need_flush) {
  1447. iommu_flush_pages(&dma_dom->domain, dma_addr, size);
  1448. dma_dom->need_flush = false;
  1449. }
  1450. }
  1451. /*
  1452. * The exported map_single function for dma_ops.
  1453. */
  1454. static dma_addr_t map_page(struct device *dev, struct page *page,
  1455. unsigned long offset, size_t size,
  1456. enum dma_data_direction dir,
  1457. struct dma_attrs *attrs)
  1458. {
  1459. unsigned long flags;
  1460. struct protection_domain *domain;
  1461. dma_addr_t addr;
  1462. u64 dma_mask;
  1463. phys_addr_t paddr = page_to_phys(page) + offset;
  1464. INC_STATS_COUNTER(cnt_map_single);
  1465. domain = get_domain(dev);
  1466. if (PTR_ERR(domain) == -EINVAL)
  1467. return (dma_addr_t)paddr;
  1468. else if (IS_ERR(domain))
  1469. return DMA_ERROR_CODE;
  1470. dma_mask = *dev->dma_mask;
  1471. spin_lock_irqsave(&domain->lock, flags);
  1472. addr = __map_single(dev, domain->priv, paddr, size, dir, false,
  1473. dma_mask);
  1474. if (addr == DMA_ERROR_CODE)
  1475. goto out;
  1476. iommu_flush_complete(domain);
  1477. out:
  1478. spin_unlock_irqrestore(&domain->lock, flags);
  1479. return addr;
  1480. }
  1481. /*
  1482. * The exported unmap_single function for dma_ops.
  1483. */
  1484. static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
  1485. enum dma_data_direction dir, struct dma_attrs *attrs)
  1486. {
  1487. unsigned long flags;
  1488. struct protection_domain *domain;
  1489. INC_STATS_COUNTER(cnt_unmap_single);
  1490. domain = get_domain(dev);
  1491. if (IS_ERR(domain))
  1492. return;
  1493. spin_lock_irqsave(&domain->lock, flags);
  1494. __unmap_single(domain->priv, dma_addr, size, dir);
  1495. iommu_flush_complete(domain);
  1496. spin_unlock_irqrestore(&domain->lock, flags);
  1497. }
  1498. /*
  1499. * This is a special map_sg function which is used if we should map a
  1500. * device which is not handled by an AMD IOMMU in the system.
  1501. */
  1502. static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
  1503. int nelems, int dir)
  1504. {
  1505. struct scatterlist *s;
  1506. int i;
  1507. for_each_sg(sglist, s, nelems, i) {
  1508. s->dma_address = (dma_addr_t)sg_phys(s);
  1509. s->dma_length = s->length;
  1510. }
  1511. return nelems;
  1512. }
  1513. /*
  1514. * The exported map_sg function for dma_ops (handles scatter-gather
  1515. * lists).
  1516. */
  1517. static int map_sg(struct device *dev, struct scatterlist *sglist,
  1518. int nelems, enum dma_data_direction dir,
  1519. struct dma_attrs *attrs)
  1520. {
  1521. unsigned long flags;
  1522. struct protection_domain *domain;
  1523. int i;
  1524. struct scatterlist *s;
  1525. phys_addr_t paddr;
  1526. int mapped_elems = 0;
  1527. u64 dma_mask;
  1528. INC_STATS_COUNTER(cnt_map_sg);
  1529. domain = get_domain(dev);
  1530. if (PTR_ERR(domain) == -EINVAL)
  1531. return map_sg_no_iommu(dev, sglist, nelems, dir);
  1532. else if (IS_ERR(domain))
  1533. return 0;
  1534. dma_mask = *dev->dma_mask;
  1535. spin_lock_irqsave(&domain->lock, flags);
  1536. for_each_sg(sglist, s, nelems, i) {
  1537. paddr = sg_phys(s);
  1538. s->dma_address = __map_single(dev, domain->priv,
  1539. paddr, s->length, dir, false,
  1540. dma_mask);
  1541. if (s->dma_address) {
  1542. s->dma_length = s->length;
  1543. mapped_elems++;
  1544. } else
  1545. goto unmap;
  1546. }
  1547. iommu_flush_complete(domain);
  1548. out:
  1549. spin_unlock_irqrestore(&domain->lock, flags);
  1550. return mapped_elems;
  1551. unmap:
  1552. for_each_sg(sglist, s, mapped_elems, i) {
  1553. if (s->dma_address)
  1554. __unmap_single(domain->priv, s->dma_address,
  1555. s->dma_length, dir);
  1556. s->dma_address = s->dma_length = 0;
  1557. }
  1558. mapped_elems = 0;
  1559. goto out;
  1560. }
  1561. /*
  1562. * The exported map_sg function for dma_ops (handles scatter-gather
  1563. * lists).
  1564. */
  1565. static void unmap_sg(struct device *dev, struct scatterlist *sglist,
  1566. int nelems, enum dma_data_direction dir,
  1567. struct dma_attrs *attrs)
  1568. {
  1569. unsigned long flags;
  1570. struct protection_domain *domain;
  1571. struct scatterlist *s;
  1572. int i;
  1573. INC_STATS_COUNTER(cnt_unmap_sg);
  1574. domain = get_domain(dev);
  1575. if (IS_ERR(domain))
  1576. return;
  1577. spin_lock_irqsave(&domain->lock, flags);
  1578. for_each_sg(sglist, s, nelems, i) {
  1579. __unmap_single(domain->priv, s->dma_address,
  1580. s->dma_length, dir);
  1581. s->dma_address = s->dma_length = 0;
  1582. }
  1583. iommu_flush_complete(domain);
  1584. spin_unlock_irqrestore(&domain->lock, flags);
  1585. }
  1586. /*
  1587. * The exported alloc_coherent function for dma_ops.
  1588. */
  1589. static void *alloc_coherent(struct device *dev, size_t size,
  1590. dma_addr_t *dma_addr, gfp_t flag)
  1591. {
  1592. unsigned long flags;
  1593. void *virt_addr;
  1594. struct protection_domain *domain;
  1595. phys_addr_t paddr;
  1596. u64 dma_mask = dev->coherent_dma_mask;
  1597. INC_STATS_COUNTER(cnt_alloc_coherent);
  1598. domain = get_domain(dev);
  1599. if (PTR_ERR(domain) == -EINVAL) {
  1600. virt_addr = (void *)__get_free_pages(flag, get_order(size));
  1601. *dma_addr = __pa(virt_addr);
  1602. return virt_addr;
  1603. } else if (IS_ERR(domain))
  1604. return NULL;
  1605. dma_mask = dev->coherent_dma_mask;
  1606. flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
  1607. flag |= __GFP_ZERO;
  1608. virt_addr = (void *)__get_free_pages(flag, get_order(size));
  1609. if (!virt_addr)
  1610. return NULL;
  1611. paddr = virt_to_phys(virt_addr);
  1612. if (!dma_mask)
  1613. dma_mask = *dev->dma_mask;
  1614. spin_lock_irqsave(&domain->lock, flags);
  1615. *dma_addr = __map_single(dev, domain->priv, paddr,
  1616. size, DMA_BIDIRECTIONAL, true, dma_mask);
  1617. if (*dma_addr == DMA_ERROR_CODE) {
  1618. spin_unlock_irqrestore(&domain->lock, flags);
  1619. goto out_free;
  1620. }
  1621. iommu_flush_complete(domain);
  1622. spin_unlock_irqrestore(&domain->lock, flags);
  1623. return virt_addr;
  1624. out_free:
  1625. free_pages((unsigned long)virt_addr, get_order(size));
  1626. return NULL;
  1627. }
  1628. /*
  1629. * The exported free_coherent function for dma_ops.
  1630. */
  1631. static void free_coherent(struct device *dev, size_t size,
  1632. void *virt_addr, dma_addr_t dma_addr)
  1633. {
  1634. unsigned long flags;
  1635. struct protection_domain *domain;
  1636. INC_STATS_COUNTER(cnt_free_coherent);
  1637. domain = get_domain(dev);
  1638. if (IS_ERR(domain))
  1639. goto free_mem;
  1640. spin_lock_irqsave(&domain->lock, flags);
  1641. __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
  1642. iommu_flush_complete(domain);
  1643. spin_unlock_irqrestore(&domain->lock, flags);
  1644. free_mem:
  1645. free_pages((unsigned long)virt_addr, get_order(size));
  1646. }
  1647. /*
  1648. * This function is called by the DMA layer to find out if we can handle a
  1649. * particular device. It is part of the dma_ops.
  1650. */
  1651. static int amd_iommu_dma_supported(struct device *dev, u64 mask)
  1652. {
  1653. return check_device(dev);
  1654. }
  1655. /*
  1656. * The function for pre-allocating protection domains.
  1657. *
  1658. * If the driver core informs the DMA layer if a driver grabs a device
  1659. * we don't need to preallocate the protection domains anymore.
  1660. * For now we have to.
  1661. */
  1662. static void prealloc_protection_domains(void)
  1663. {
  1664. struct pci_dev *dev = NULL;
  1665. struct dma_ops_domain *dma_dom;
  1666. u16 devid;
  1667. while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
  1668. /* Do we handle this device? */
  1669. if (!check_device(&dev->dev))
  1670. continue;
  1671. /* Is there already any domain for it? */
  1672. if (domain_for_device(&dev->dev))
  1673. continue;
  1674. devid = get_device_id(&dev->dev);
  1675. dma_dom = dma_ops_domain_alloc();
  1676. if (!dma_dom)
  1677. continue;
  1678. init_unity_mappings_for_device(dma_dom, devid);
  1679. dma_dom->target_dev = devid;
  1680. attach_device(&dev->dev, &dma_dom->domain);
  1681. list_add_tail(&dma_dom->list, &iommu_pd_list);
  1682. }
  1683. }
  1684. static struct dma_map_ops amd_iommu_dma_ops = {
  1685. .alloc_coherent = alloc_coherent,
  1686. .free_coherent = free_coherent,
  1687. .map_page = map_page,
  1688. .unmap_page = unmap_page,
  1689. .map_sg = map_sg,
  1690. .unmap_sg = unmap_sg,
  1691. .dma_supported = amd_iommu_dma_supported,
  1692. };
  1693. /*
  1694. * The function which clues the AMD IOMMU driver into dma_ops.
  1695. */
  1696. int __init amd_iommu_init_dma_ops(void)
  1697. {
  1698. struct amd_iommu *iommu;
  1699. int ret;
  1700. /*
  1701. * first allocate a default protection domain for every IOMMU we
  1702. * found in the system. Devices not assigned to any other
  1703. * protection domain will be assigned to the default one.
  1704. */
  1705. for_each_iommu(iommu) {
  1706. iommu->default_dom = dma_ops_domain_alloc();
  1707. if (iommu->default_dom == NULL)
  1708. return -ENOMEM;
  1709. iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
  1710. ret = iommu_init_unity_mappings(iommu);
  1711. if (ret)
  1712. goto free_domains;
  1713. }
  1714. /*
  1715. * If device isolation is enabled, pre-allocate the protection
  1716. * domains for each device.
  1717. */
  1718. if (amd_iommu_isolate)
  1719. prealloc_protection_domains();
  1720. iommu_detected = 1;
  1721. swiotlb = 0;
  1722. #ifdef CONFIG_GART_IOMMU
  1723. gart_iommu_aperture_disabled = 1;
  1724. gart_iommu_aperture = 0;
  1725. #endif
  1726. /* Make the driver finally visible to the drivers */
  1727. dma_ops = &amd_iommu_dma_ops;
  1728. register_iommu(&amd_iommu_ops);
  1729. bus_register_notifier(&pci_bus_type, &device_nb);
  1730. amd_iommu_stats_init();
  1731. return 0;
  1732. free_domains:
  1733. for_each_iommu(iommu) {
  1734. if (iommu->default_dom)
  1735. dma_ops_domain_free(iommu->default_dom);
  1736. }
  1737. return ret;
  1738. }
  1739. /*****************************************************************************
  1740. *
  1741. * The following functions belong to the exported interface of AMD IOMMU
  1742. *
  1743. * This interface allows access to lower level functions of the IOMMU
  1744. * like protection domain handling and assignement of devices to domains
  1745. * which is not possible with the dma_ops interface.
  1746. *
  1747. *****************************************************************************/
  1748. static void cleanup_domain(struct protection_domain *domain)
  1749. {
  1750. unsigned long flags;
  1751. u16 devid;
  1752. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1753. for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
  1754. if (amd_iommu_pd_table[devid] == domain)
  1755. clear_dte_entry(devid);
  1756. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1757. }
  1758. static void protection_domain_free(struct protection_domain *domain)
  1759. {
  1760. if (!domain)
  1761. return;
  1762. del_domain_from_list(domain);
  1763. if (domain->id)
  1764. domain_id_free(domain->id);
  1765. kfree(domain);
  1766. }
  1767. static struct protection_domain *protection_domain_alloc(void)
  1768. {
  1769. struct protection_domain *domain;
  1770. domain = kzalloc(sizeof(*domain), GFP_KERNEL);
  1771. if (!domain)
  1772. return NULL;
  1773. spin_lock_init(&domain->lock);
  1774. domain->id = domain_id_alloc();
  1775. if (!domain->id)
  1776. goto out_err;
  1777. add_domain_to_list(domain);
  1778. return domain;
  1779. out_err:
  1780. kfree(domain);
  1781. return NULL;
  1782. }
  1783. static int amd_iommu_domain_init(struct iommu_domain *dom)
  1784. {
  1785. struct protection_domain *domain;
  1786. domain = protection_domain_alloc();
  1787. if (!domain)
  1788. goto out_free;
  1789. domain->mode = PAGE_MODE_3_LEVEL;
  1790. domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
  1791. if (!domain->pt_root)
  1792. goto out_free;
  1793. dom->priv = domain;
  1794. return 0;
  1795. out_free:
  1796. protection_domain_free(domain);
  1797. return -ENOMEM;
  1798. }
  1799. static void amd_iommu_domain_destroy(struct iommu_domain *dom)
  1800. {
  1801. struct protection_domain *domain = dom->priv;
  1802. if (!domain)
  1803. return;
  1804. if (domain->dev_cnt > 0)
  1805. cleanup_domain(domain);
  1806. BUG_ON(domain->dev_cnt != 0);
  1807. free_pagetable(domain);
  1808. domain_id_free(domain->id);
  1809. kfree(domain);
  1810. dom->priv = NULL;
  1811. }
  1812. static void amd_iommu_detach_device(struct iommu_domain *dom,
  1813. struct device *dev)
  1814. {
  1815. struct amd_iommu *iommu;
  1816. u16 devid;
  1817. if (!check_device(dev))
  1818. return;
  1819. devid = get_device_id(dev);
  1820. if (amd_iommu_pd_table[devid] != NULL)
  1821. detach_device(dev);
  1822. iommu = amd_iommu_rlookup_table[devid];
  1823. if (!iommu)
  1824. return;
  1825. iommu_queue_inv_dev_entry(iommu, devid);
  1826. iommu_completion_wait(iommu);
  1827. }
  1828. static int amd_iommu_attach_device(struct iommu_domain *dom,
  1829. struct device *dev)
  1830. {
  1831. struct protection_domain *domain = dom->priv;
  1832. struct protection_domain *old_domain;
  1833. struct amd_iommu *iommu;
  1834. int ret;
  1835. u16 devid;
  1836. if (!check_device(dev))
  1837. return -EINVAL;
  1838. devid = get_device_id(dev);
  1839. iommu = amd_iommu_rlookup_table[devid];
  1840. if (!iommu)
  1841. return -EINVAL;
  1842. old_domain = amd_iommu_pd_table[devid];
  1843. if (old_domain)
  1844. detach_device(dev);
  1845. ret = attach_device(dev, domain);
  1846. iommu_completion_wait(iommu);
  1847. return ret;
  1848. }
  1849. static int amd_iommu_map_range(struct iommu_domain *dom,
  1850. unsigned long iova, phys_addr_t paddr,
  1851. size_t size, int iommu_prot)
  1852. {
  1853. struct protection_domain *domain = dom->priv;
  1854. unsigned long i, npages = iommu_num_pages(paddr, size, PAGE_SIZE);
  1855. int prot = 0;
  1856. int ret;
  1857. if (iommu_prot & IOMMU_READ)
  1858. prot |= IOMMU_PROT_IR;
  1859. if (iommu_prot & IOMMU_WRITE)
  1860. prot |= IOMMU_PROT_IW;
  1861. iova &= PAGE_MASK;
  1862. paddr &= PAGE_MASK;
  1863. for (i = 0; i < npages; ++i) {
  1864. ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
  1865. if (ret)
  1866. return ret;
  1867. iova += PAGE_SIZE;
  1868. paddr += PAGE_SIZE;
  1869. }
  1870. return 0;
  1871. }
  1872. static void amd_iommu_unmap_range(struct iommu_domain *dom,
  1873. unsigned long iova, size_t size)
  1874. {
  1875. struct protection_domain *domain = dom->priv;
  1876. unsigned long i, npages = iommu_num_pages(iova, size, PAGE_SIZE);
  1877. iova &= PAGE_MASK;
  1878. for (i = 0; i < npages; ++i) {
  1879. iommu_unmap_page(domain, iova, PM_MAP_4k);
  1880. iova += PAGE_SIZE;
  1881. }
  1882. iommu_flush_tlb_pde(domain);
  1883. }
  1884. static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
  1885. unsigned long iova)
  1886. {
  1887. struct protection_domain *domain = dom->priv;
  1888. unsigned long offset = iova & ~PAGE_MASK;
  1889. phys_addr_t paddr;
  1890. u64 *pte;
  1891. pte = fetch_pte(domain, iova, PM_MAP_4k);
  1892. if (!pte || !IOMMU_PTE_PRESENT(*pte))
  1893. return 0;
  1894. paddr = *pte & IOMMU_PAGE_MASK;
  1895. paddr |= offset;
  1896. return paddr;
  1897. }
  1898. static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
  1899. unsigned long cap)
  1900. {
  1901. return 0;
  1902. }
  1903. static struct iommu_ops amd_iommu_ops = {
  1904. .domain_init = amd_iommu_domain_init,
  1905. .domain_destroy = amd_iommu_domain_destroy,
  1906. .attach_dev = amd_iommu_attach_device,
  1907. .detach_dev = amd_iommu_detach_device,
  1908. .map = amd_iommu_map_range,
  1909. .unmap = amd_iommu_unmap_range,
  1910. .iova_to_phys = amd_iommu_iova_to_phys,
  1911. .domain_has_cap = amd_iommu_domain_has_cap,
  1912. };
  1913. /*****************************************************************************
  1914. *
  1915. * The next functions do a basic initialization of IOMMU for pass through
  1916. * mode
  1917. *
  1918. * In passthrough mode the IOMMU is initialized and enabled but not used for
  1919. * DMA-API translation.
  1920. *
  1921. *****************************************************************************/
  1922. int __init amd_iommu_init_passthrough(void)
  1923. {
  1924. struct amd_iommu *iommu;
  1925. struct pci_dev *dev = NULL;
  1926. u16 devid;
  1927. /* allocate passthroug domain */
  1928. pt_domain = protection_domain_alloc();
  1929. if (!pt_domain)
  1930. return -ENOMEM;
  1931. pt_domain->mode |= PAGE_MODE_NONE;
  1932. while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
  1933. if (!check_device(&dev->dev))
  1934. continue;
  1935. devid = get_device_id(&dev->dev);
  1936. iommu = amd_iommu_rlookup_table[devid];
  1937. if (!iommu)
  1938. continue;
  1939. attach_device(&dev->dev, pt_domain);
  1940. }
  1941. pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
  1942. return 0;
  1943. }