amd_iommu_init.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943
  1. /*
  2. * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  3. * Author: Joerg Roedel <joerg.roedel@amd.com>
  4. * Leo Duran <leo.duran@amd.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/pci.h>
  20. #include <linux/acpi.h>
  21. #include <linux/list.h>
  22. #include <linux/slab.h>
  23. #include <linux/syscore_ops.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/msi.h>
  26. #include <linux/amd-iommu.h>
  27. #include <linux/export.h>
  28. #include <linux/acpi.h>
  29. #include <acpi/acpi.h>
  30. #include <asm/pci-direct.h>
  31. #include <asm/iommu.h>
  32. #include <asm/gart.h>
  33. #include <asm/x86_init.h>
  34. #include <asm/iommu_table.h>
  35. #include "amd_iommu_proto.h"
  36. #include "amd_iommu_types.h"
  37. #include "irq_remapping.h"
  38. /*
  39. * definitions for the ACPI scanning code
  40. */
  41. #define IVRS_HEADER_LENGTH 48
  42. #define ACPI_IVHD_TYPE 0x10
  43. #define ACPI_IVMD_TYPE_ALL 0x20
  44. #define ACPI_IVMD_TYPE 0x21
  45. #define ACPI_IVMD_TYPE_RANGE 0x22
  46. #define IVHD_DEV_ALL 0x01
  47. #define IVHD_DEV_SELECT 0x02
  48. #define IVHD_DEV_SELECT_RANGE_START 0x03
  49. #define IVHD_DEV_RANGE_END 0x04
  50. #define IVHD_DEV_ALIAS 0x42
  51. #define IVHD_DEV_ALIAS_RANGE 0x43
  52. #define IVHD_DEV_EXT_SELECT 0x46
  53. #define IVHD_DEV_EXT_SELECT_RANGE 0x47
  54. #define IVHD_DEV_SPECIAL 0x48
  55. #define IVHD_SPECIAL_IOAPIC 1
  56. #define IVHD_SPECIAL_HPET 2
  57. #define IVHD_FLAG_HT_TUN_EN_MASK 0x01
  58. #define IVHD_FLAG_PASSPW_EN_MASK 0x02
  59. #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
  60. #define IVHD_FLAG_ISOC_EN_MASK 0x08
  61. #define IVMD_FLAG_EXCL_RANGE 0x08
  62. #define IVMD_FLAG_UNITY_MAP 0x01
  63. #define ACPI_DEVFLAG_INITPASS 0x01
  64. #define ACPI_DEVFLAG_EXTINT 0x02
  65. #define ACPI_DEVFLAG_NMI 0x04
  66. #define ACPI_DEVFLAG_SYSMGT1 0x10
  67. #define ACPI_DEVFLAG_SYSMGT2 0x20
  68. #define ACPI_DEVFLAG_LINT0 0x40
  69. #define ACPI_DEVFLAG_LINT1 0x80
  70. #define ACPI_DEVFLAG_ATSDIS 0x10000000
  71. /*
  72. * ACPI table definitions
  73. *
  74. * These data structures are laid over the table to parse the important values
  75. * out of it.
  76. */
  77. /*
  78. * structure describing one IOMMU in the ACPI table. Typically followed by one
  79. * or more ivhd_entrys.
  80. */
  81. struct ivhd_header {
  82. u8 type;
  83. u8 flags;
  84. u16 length;
  85. u16 devid;
  86. u16 cap_ptr;
  87. u64 mmio_phys;
  88. u16 pci_seg;
  89. u16 info;
  90. u32 reserved;
  91. } __attribute__((packed));
  92. /*
  93. * A device entry describing which devices a specific IOMMU translates and
  94. * which requestor ids they use.
  95. */
  96. struct ivhd_entry {
  97. u8 type;
  98. u16 devid;
  99. u8 flags;
  100. u32 ext;
  101. } __attribute__((packed));
  102. /*
  103. * An AMD IOMMU memory definition structure. It defines things like exclusion
  104. * ranges for devices and regions that should be unity mapped.
  105. */
  106. struct ivmd_header {
  107. u8 type;
  108. u8 flags;
  109. u16 length;
  110. u16 devid;
  111. u16 aux;
  112. u64 resv;
  113. u64 range_start;
  114. u64 range_length;
  115. } __attribute__((packed));
  116. bool amd_iommu_dump;
  117. bool amd_iommu_irq_remap __read_mostly;
  118. static bool amd_iommu_detected;
  119. static bool __initdata amd_iommu_disabled;
  120. u16 amd_iommu_last_bdf; /* largest PCI device id we have
  121. to handle */
  122. LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
  123. we find in ACPI */
  124. u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */
  125. LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
  126. system */
  127. /* Array to assign indices to IOMMUs*/
  128. struct amd_iommu *amd_iommus[MAX_IOMMUS];
  129. int amd_iommus_present;
  130. /* IOMMUs have a non-present cache? */
  131. bool amd_iommu_np_cache __read_mostly;
  132. bool amd_iommu_iotlb_sup __read_mostly = true;
  133. u32 amd_iommu_max_pasids __read_mostly = ~0;
  134. bool amd_iommu_v2_present __read_mostly;
  135. bool amd_iommu_force_isolation __read_mostly;
  136. /*
  137. * List of protection domains - used during resume
  138. */
  139. LIST_HEAD(amd_iommu_pd_list);
  140. spinlock_t amd_iommu_pd_lock;
  141. /*
  142. * Pointer to the device table which is shared by all AMD IOMMUs
  143. * it is indexed by the PCI device id or the HT unit id and contains
  144. * information about the domain the device belongs to as well as the
  145. * page table root pointer.
  146. */
  147. struct dev_table_entry *amd_iommu_dev_table;
  148. /*
  149. * The alias table is a driver specific data structure which contains the
  150. * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
  151. * More than one device can share the same requestor id.
  152. */
  153. u16 *amd_iommu_alias_table;
  154. /*
  155. * The rlookup table is used to find the IOMMU which is responsible
  156. * for a specific device. It is also indexed by the PCI device id.
  157. */
  158. struct amd_iommu **amd_iommu_rlookup_table;
  159. /*
  160. * This table is used to find the irq remapping table for a given device id
  161. * quickly.
  162. */
  163. struct irq_remap_table **irq_lookup_table;
  164. /*
  165. * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap
  166. * to know which ones are already in use.
  167. */
  168. unsigned long *amd_iommu_pd_alloc_bitmap;
  169. static u32 dev_table_size; /* size of the device table */
  170. static u32 alias_table_size; /* size of the alias table */
  171. static u32 rlookup_table_size; /* size if the rlookup table */
  172. enum iommu_init_state {
  173. IOMMU_START_STATE,
  174. IOMMU_IVRS_DETECTED,
  175. IOMMU_ACPI_FINISHED,
  176. IOMMU_ENABLED,
  177. IOMMU_PCI_INIT,
  178. IOMMU_INTERRUPTS_EN,
  179. IOMMU_DMA_OPS,
  180. IOMMU_INITIALIZED,
  181. IOMMU_NOT_FOUND,
  182. IOMMU_INIT_ERROR,
  183. };
  184. static enum iommu_init_state init_state = IOMMU_START_STATE;
  185. static int amd_iommu_enable_interrupts(void);
  186. static int __init iommu_go_to_state(enum iommu_init_state state);
  187. static inline void update_last_devid(u16 devid)
  188. {
  189. if (devid > amd_iommu_last_bdf)
  190. amd_iommu_last_bdf = devid;
  191. }
  192. static inline unsigned long tbl_size(int entry_size)
  193. {
  194. unsigned shift = PAGE_SHIFT +
  195. get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
  196. return 1UL << shift;
  197. }
  198. /* Access to l1 and l2 indexed register spaces */
  199. static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
  200. {
  201. u32 val;
  202. pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
  203. pci_read_config_dword(iommu->dev, 0xfc, &val);
  204. return val;
  205. }
  206. static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
  207. {
  208. pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
  209. pci_write_config_dword(iommu->dev, 0xfc, val);
  210. pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
  211. }
  212. static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
  213. {
  214. u32 val;
  215. pci_write_config_dword(iommu->dev, 0xf0, address);
  216. pci_read_config_dword(iommu->dev, 0xf4, &val);
  217. return val;
  218. }
  219. static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
  220. {
  221. pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
  222. pci_write_config_dword(iommu->dev, 0xf4, val);
  223. }
  224. /****************************************************************************
  225. *
  226. * AMD IOMMU MMIO register space handling functions
  227. *
  228. * These functions are used to program the IOMMU device registers in
  229. * MMIO space required for that driver.
  230. *
  231. ****************************************************************************/
  232. /*
  233. * This function set the exclusion range in the IOMMU. DMA accesses to the
  234. * exclusion range are passed through untranslated
  235. */
  236. static void iommu_set_exclusion_range(struct amd_iommu *iommu)
  237. {
  238. u64 start = iommu->exclusion_start & PAGE_MASK;
  239. u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
  240. u64 entry;
  241. if (!iommu->exclusion_start)
  242. return;
  243. entry = start | MMIO_EXCL_ENABLE_MASK;
  244. memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
  245. &entry, sizeof(entry));
  246. entry = limit;
  247. memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
  248. &entry, sizeof(entry));
  249. }
  250. /* Programs the physical address of the device table into the IOMMU hardware */
  251. static void iommu_set_device_table(struct amd_iommu *iommu)
  252. {
  253. u64 entry;
  254. BUG_ON(iommu->mmio_base == NULL);
  255. entry = virt_to_phys(amd_iommu_dev_table);
  256. entry |= (dev_table_size >> 12) - 1;
  257. memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
  258. &entry, sizeof(entry));
  259. }
  260. /* Generic functions to enable/disable certain features of the IOMMU. */
  261. static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
  262. {
  263. u32 ctrl;
  264. ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
  265. ctrl |= (1 << bit);
  266. writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
  267. }
  268. static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
  269. {
  270. u32 ctrl;
  271. ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
  272. ctrl &= ~(1 << bit);
  273. writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
  274. }
  275. static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
  276. {
  277. u32 ctrl;
  278. ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
  279. ctrl &= ~CTRL_INV_TO_MASK;
  280. ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
  281. writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
  282. }
  283. /* Function to enable the hardware */
  284. static void iommu_enable(struct amd_iommu *iommu)
  285. {
  286. iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
  287. }
  288. static void iommu_disable(struct amd_iommu *iommu)
  289. {
  290. /* Disable command buffer */
  291. iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
  292. /* Disable event logging and event interrupts */
  293. iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
  294. iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
  295. /* Disable IOMMU hardware itself */
  296. iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
  297. }
  298. /*
  299. * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
  300. * the system has one.
  301. */
  302. static u8 __iomem * __init iommu_map_mmio_space(u64 address)
  303. {
  304. if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {
  305. pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",
  306. address);
  307. pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
  308. return NULL;
  309. }
  310. return (u8 __iomem *)ioremap_nocache(address, MMIO_REGION_LENGTH);
  311. }
  312. static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
  313. {
  314. if (iommu->mmio_base)
  315. iounmap(iommu->mmio_base);
  316. release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
  317. }
  318. /****************************************************************************
  319. *
  320. * The functions below belong to the first pass of AMD IOMMU ACPI table
  321. * parsing. In this pass we try to find out the highest device id this
  322. * code has to handle. Upon this information the size of the shared data
  323. * structures is determined later.
  324. *
  325. ****************************************************************************/
  326. /*
  327. * This function calculates the length of a given IVHD entry
  328. */
  329. static inline int ivhd_entry_length(u8 *ivhd)
  330. {
  331. return 0x04 << (*ivhd >> 6);
  332. }
  333. /*
  334. * This function reads the last device id the IOMMU has to handle from the PCI
  335. * capability header for this IOMMU
  336. */
  337. static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
  338. {
  339. u32 cap;
  340. cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
  341. update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
  342. return 0;
  343. }
  344. /*
  345. * After reading the highest device id from the IOMMU PCI capability header
  346. * this function looks if there is a higher device id defined in the ACPI table
  347. */
  348. static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
  349. {
  350. u8 *p = (void *)h, *end = (void *)h;
  351. struct ivhd_entry *dev;
  352. p += sizeof(*h);
  353. end += h->length;
  354. find_last_devid_on_pci(PCI_BUS(h->devid),
  355. PCI_SLOT(h->devid),
  356. PCI_FUNC(h->devid),
  357. h->cap_ptr);
  358. while (p < end) {
  359. dev = (struct ivhd_entry *)p;
  360. switch (dev->type) {
  361. case IVHD_DEV_SELECT:
  362. case IVHD_DEV_RANGE_END:
  363. case IVHD_DEV_ALIAS:
  364. case IVHD_DEV_EXT_SELECT:
  365. /* all the above subfield types refer to device ids */
  366. update_last_devid(dev->devid);
  367. break;
  368. default:
  369. break;
  370. }
  371. p += ivhd_entry_length(p);
  372. }
  373. WARN_ON(p != end);
  374. return 0;
  375. }
  376. /*
  377. * Iterate over all IVHD entries in the ACPI table and find the highest device
  378. * id which we need to handle. This is the first of three functions which parse
  379. * the ACPI table. So we check the checksum here.
  380. */
  381. static int __init find_last_devid_acpi(struct acpi_table_header *table)
  382. {
  383. int i;
  384. u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
  385. struct ivhd_header *h;
  386. /*
  387. * Validate checksum here so we don't need to do it when
  388. * we actually parse the table
  389. */
  390. for (i = 0; i < table->length; ++i)
  391. checksum += p[i];
  392. if (checksum != 0)
  393. /* ACPI table corrupt */
  394. return -ENODEV;
  395. p += IVRS_HEADER_LENGTH;
  396. end += table->length;
  397. while (p < end) {
  398. h = (struct ivhd_header *)p;
  399. switch (h->type) {
  400. case ACPI_IVHD_TYPE:
  401. find_last_devid_from_ivhd(h);
  402. break;
  403. default:
  404. break;
  405. }
  406. p += h->length;
  407. }
  408. WARN_ON(p != end);
  409. return 0;
  410. }
  411. /****************************************************************************
  412. *
  413. * The following functions belong the the code path which parses the ACPI table
  414. * the second time. In this ACPI parsing iteration we allocate IOMMU specific
  415. * data structures, initialize the device/alias/rlookup table and also
  416. * basically initialize the hardware.
  417. *
  418. ****************************************************************************/
  419. /*
  420. * Allocates the command buffer. This buffer is per AMD IOMMU. We can
  421. * write commands to that buffer later and the IOMMU will execute them
  422. * asynchronously
  423. */
  424. static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
  425. {
  426. u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  427. get_order(CMD_BUFFER_SIZE));
  428. if (cmd_buf == NULL)
  429. return NULL;
  430. iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;
  431. return cmd_buf;
  432. }
  433. /*
  434. * This function resets the command buffer if the IOMMU stopped fetching
  435. * commands from it.
  436. */
  437. void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
  438. {
  439. iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
  440. writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  441. writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  442. iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
  443. }
  444. /*
  445. * This function writes the command buffer address to the hardware and
  446. * enables it.
  447. */
  448. static void iommu_enable_command_buffer(struct amd_iommu *iommu)
  449. {
  450. u64 entry;
  451. BUG_ON(iommu->cmd_buf == NULL);
  452. entry = (u64)virt_to_phys(iommu->cmd_buf);
  453. entry |= MMIO_CMD_SIZE_512;
  454. memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
  455. &entry, sizeof(entry));
  456. amd_iommu_reset_cmd_buffer(iommu);
  457. iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);
  458. }
  459. static void __init free_command_buffer(struct amd_iommu *iommu)
  460. {
  461. free_pages((unsigned long)iommu->cmd_buf,
  462. get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));
  463. }
  464. /* allocates the memory where the IOMMU will log its events to */
  465. static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
  466. {
  467. iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  468. get_order(EVT_BUFFER_SIZE));
  469. if (iommu->evt_buf == NULL)
  470. return NULL;
  471. iommu->evt_buf_size = EVT_BUFFER_SIZE;
  472. return iommu->evt_buf;
  473. }
  474. static void iommu_enable_event_buffer(struct amd_iommu *iommu)
  475. {
  476. u64 entry;
  477. BUG_ON(iommu->evt_buf == NULL);
  478. entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
  479. memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
  480. &entry, sizeof(entry));
  481. /* set head and tail to zero manually */
  482. writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  483. writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
  484. iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
  485. }
  486. static void __init free_event_buffer(struct amd_iommu *iommu)
  487. {
  488. free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
  489. }
  490. /* allocates the memory where the IOMMU will log its events to */
  491. static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
  492. {
  493. iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  494. get_order(PPR_LOG_SIZE));
  495. if (iommu->ppr_log == NULL)
  496. return NULL;
  497. return iommu->ppr_log;
  498. }
  499. static void iommu_enable_ppr_log(struct amd_iommu *iommu)
  500. {
  501. u64 entry;
  502. if (iommu->ppr_log == NULL)
  503. return;
  504. entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
  505. memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
  506. &entry, sizeof(entry));
  507. /* set head and tail to zero manually */
  508. writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  509. writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
  510. iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
  511. iommu_feature_enable(iommu, CONTROL_PPR_EN);
  512. }
  513. static void __init free_ppr_log(struct amd_iommu *iommu)
  514. {
  515. if (iommu->ppr_log == NULL)
  516. return;
  517. free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
  518. }
  519. static void iommu_enable_gt(struct amd_iommu *iommu)
  520. {
  521. if (!iommu_feature(iommu, FEATURE_GT))
  522. return;
  523. iommu_feature_enable(iommu, CONTROL_GT_EN);
  524. }
  525. /* sets a specific bit in the device table entry. */
  526. static void set_dev_entry_bit(u16 devid, u8 bit)
  527. {
  528. int i = (bit >> 6) & 0x03;
  529. int _bit = bit & 0x3f;
  530. amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);
  531. }
  532. static int get_dev_entry_bit(u16 devid, u8 bit)
  533. {
  534. int i = (bit >> 6) & 0x03;
  535. int _bit = bit & 0x3f;
  536. return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
  537. }
  538. void amd_iommu_apply_erratum_63(u16 devid)
  539. {
  540. int sysmgt;
  541. sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |
  542. (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
  543. if (sysmgt == 0x01)
  544. set_dev_entry_bit(devid, DEV_ENTRY_IW);
  545. }
  546. /* Writes the specific IOMMU for a device into the rlookup table */
  547. static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
  548. {
  549. amd_iommu_rlookup_table[devid] = iommu;
  550. }
  551. /*
  552. * This function takes the device specific flags read from the ACPI
  553. * table and sets up the device table entry with that information
  554. */
  555. static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
  556. u16 devid, u32 flags, u32 ext_flags)
  557. {
  558. if (flags & ACPI_DEVFLAG_INITPASS)
  559. set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
  560. if (flags & ACPI_DEVFLAG_EXTINT)
  561. set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
  562. if (flags & ACPI_DEVFLAG_NMI)
  563. set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
  564. if (flags & ACPI_DEVFLAG_SYSMGT1)
  565. set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
  566. if (flags & ACPI_DEVFLAG_SYSMGT2)
  567. set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
  568. if (flags & ACPI_DEVFLAG_LINT0)
  569. set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
  570. if (flags & ACPI_DEVFLAG_LINT1)
  571. set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
  572. amd_iommu_apply_erratum_63(devid);
  573. set_iommu_for_device(iommu, devid);
  574. }
  575. static int add_special_device(u8 type, u8 id, u16 devid)
  576. {
  577. struct devid_map *entry;
  578. struct list_head *list;
  579. if (type != IVHD_SPECIAL_IOAPIC && type != IVHD_SPECIAL_HPET)
  580. return -EINVAL;
  581. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  582. if (!entry)
  583. return -ENOMEM;
  584. entry->id = id;
  585. entry->devid = devid;
  586. if (type == IVHD_SPECIAL_IOAPIC)
  587. list = &ioapic_map;
  588. else
  589. list = &hpet_map;
  590. list_add_tail(&entry->list, list);
  591. return 0;
  592. }
  593. /*
  594. * Reads the device exclusion range from ACPI and initialize IOMMU with
  595. * it
  596. */
  597. static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
  598. {
  599. struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
  600. if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
  601. return;
  602. if (iommu) {
  603. /*
  604. * We only can configure exclusion ranges per IOMMU, not
  605. * per device. But we can enable the exclusion range per
  606. * device. This is done here
  607. */
  608. set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
  609. iommu->exclusion_start = m->range_start;
  610. iommu->exclusion_length = m->range_length;
  611. }
  612. }
  613. /*
  614. * Takes a pointer to an AMD IOMMU entry in the ACPI table and
  615. * initializes the hardware and our data structures with it.
  616. */
  617. static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
  618. struct ivhd_header *h)
  619. {
  620. u8 *p = (u8 *)h;
  621. u8 *end = p, flags = 0;
  622. u16 devid = 0, devid_start = 0, devid_to = 0;
  623. u32 dev_i, ext_flags = 0;
  624. bool alias = false;
  625. struct ivhd_entry *e;
  626. /*
  627. * First save the recommended feature enable bits from ACPI
  628. */
  629. iommu->acpi_flags = h->flags;
  630. /*
  631. * Done. Now parse the device entries
  632. */
  633. p += sizeof(struct ivhd_header);
  634. end += h->length;
  635. while (p < end) {
  636. e = (struct ivhd_entry *)p;
  637. switch (e->type) {
  638. case IVHD_DEV_ALL:
  639. DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
  640. " last device %02x:%02x.%x flags: %02x\n",
  641. PCI_BUS(iommu->first_device),
  642. PCI_SLOT(iommu->first_device),
  643. PCI_FUNC(iommu->first_device),
  644. PCI_BUS(iommu->last_device),
  645. PCI_SLOT(iommu->last_device),
  646. PCI_FUNC(iommu->last_device),
  647. e->flags);
  648. for (dev_i = iommu->first_device;
  649. dev_i <= iommu->last_device; ++dev_i)
  650. set_dev_entry_from_acpi(iommu, dev_i,
  651. e->flags, 0);
  652. break;
  653. case IVHD_DEV_SELECT:
  654. DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
  655. "flags: %02x\n",
  656. PCI_BUS(e->devid),
  657. PCI_SLOT(e->devid),
  658. PCI_FUNC(e->devid),
  659. e->flags);
  660. devid = e->devid;
  661. set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
  662. break;
  663. case IVHD_DEV_SELECT_RANGE_START:
  664. DUMP_printk(" DEV_SELECT_RANGE_START\t "
  665. "devid: %02x:%02x.%x flags: %02x\n",
  666. PCI_BUS(e->devid),
  667. PCI_SLOT(e->devid),
  668. PCI_FUNC(e->devid),
  669. e->flags);
  670. devid_start = e->devid;
  671. flags = e->flags;
  672. ext_flags = 0;
  673. alias = false;
  674. break;
  675. case IVHD_DEV_ALIAS:
  676. DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
  677. "flags: %02x devid_to: %02x:%02x.%x\n",
  678. PCI_BUS(e->devid),
  679. PCI_SLOT(e->devid),
  680. PCI_FUNC(e->devid),
  681. e->flags,
  682. PCI_BUS(e->ext >> 8),
  683. PCI_SLOT(e->ext >> 8),
  684. PCI_FUNC(e->ext >> 8));
  685. devid = e->devid;
  686. devid_to = e->ext >> 8;
  687. set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
  688. set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
  689. amd_iommu_alias_table[devid] = devid_to;
  690. break;
  691. case IVHD_DEV_ALIAS_RANGE:
  692. DUMP_printk(" DEV_ALIAS_RANGE\t\t "
  693. "devid: %02x:%02x.%x flags: %02x "
  694. "devid_to: %02x:%02x.%x\n",
  695. PCI_BUS(e->devid),
  696. PCI_SLOT(e->devid),
  697. PCI_FUNC(e->devid),
  698. e->flags,
  699. PCI_BUS(e->ext >> 8),
  700. PCI_SLOT(e->ext >> 8),
  701. PCI_FUNC(e->ext >> 8));
  702. devid_start = e->devid;
  703. flags = e->flags;
  704. devid_to = e->ext >> 8;
  705. ext_flags = 0;
  706. alias = true;
  707. break;
  708. case IVHD_DEV_EXT_SELECT:
  709. DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
  710. "flags: %02x ext: %08x\n",
  711. PCI_BUS(e->devid),
  712. PCI_SLOT(e->devid),
  713. PCI_FUNC(e->devid),
  714. e->flags, e->ext);
  715. devid = e->devid;
  716. set_dev_entry_from_acpi(iommu, devid, e->flags,
  717. e->ext);
  718. break;
  719. case IVHD_DEV_EXT_SELECT_RANGE:
  720. DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
  721. "%02x:%02x.%x flags: %02x ext: %08x\n",
  722. PCI_BUS(e->devid),
  723. PCI_SLOT(e->devid),
  724. PCI_FUNC(e->devid),
  725. e->flags, e->ext);
  726. devid_start = e->devid;
  727. flags = e->flags;
  728. ext_flags = e->ext;
  729. alias = false;
  730. break;
  731. case IVHD_DEV_RANGE_END:
  732. DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
  733. PCI_BUS(e->devid),
  734. PCI_SLOT(e->devid),
  735. PCI_FUNC(e->devid));
  736. devid = e->devid;
  737. for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
  738. if (alias) {
  739. amd_iommu_alias_table[dev_i] = devid_to;
  740. set_dev_entry_from_acpi(iommu,
  741. devid_to, flags, ext_flags);
  742. }
  743. set_dev_entry_from_acpi(iommu, dev_i,
  744. flags, ext_flags);
  745. }
  746. break;
  747. case IVHD_DEV_SPECIAL: {
  748. u8 handle, type;
  749. const char *var;
  750. u16 devid;
  751. int ret;
  752. handle = e->ext & 0xff;
  753. devid = (e->ext >> 8) & 0xffff;
  754. type = (e->ext >> 24) & 0xff;
  755. if (type == IVHD_SPECIAL_IOAPIC)
  756. var = "IOAPIC";
  757. else if (type == IVHD_SPECIAL_HPET)
  758. var = "HPET";
  759. else
  760. var = "UNKNOWN";
  761. DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
  762. var, (int)handle,
  763. PCI_BUS(devid),
  764. PCI_SLOT(devid),
  765. PCI_FUNC(devid));
  766. set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
  767. ret = add_special_device(type, handle, devid);
  768. if (ret)
  769. return ret;
  770. break;
  771. }
  772. default:
  773. break;
  774. }
  775. p += ivhd_entry_length(p);
  776. }
  777. return 0;
  778. }
  779. /* Initializes the device->iommu mapping for the driver */
  780. static int __init init_iommu_devices(struct amd_iommu *iommu)
  781. {
  782. u32 i;
  783. for (i = iommu->first_device; i <= iommu->last_device; ++i)
  784. set_iommu_for_device(iommu, i);
  785. return 0;
  786. }
  787. static void __init free_iommu_one(struct amd_iommu *iommu)
  788. {
  789. free_command_buffer(iommu);
  790. free_event_buffer(iommu);
  791. free_ppr_log(iommu);
  792. iommu_unmap_mmio_space(iommu);
  793. }
  794. static void __init free_iommu_all(void)
  795. {
  796. struct amd_iommu *iommu, *next;
  797. for_each_iommu_safe(iommu, next) {
  798. list_del(&iommu->list);
  799. free_iommu_one(iommu);
  800. kfree(iommu);
  801. }
  802. }
  803. /*
  804. * This function clues the initialization function for one IOMMU
  805. * together and also allocates the command buffer and programs the
  806. * hardware. It does NOT enable the IOMMU. This is done afterwards.
  807. */
  808. static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
  809. {
  810. int ret;
  811. spin_lock_init(&iommu->lock);
  812. /* Add IOMMU to internal data structures */
  813. list_add_tail(&iommu->list, &amd_iommu_list);
  814. iommu->index = amd_iommus_present++;
  815. if (unlikely(iommu->index >= MAX_IOMMUS)) {
  816. WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
  817. return -ENOSYS;
  818. }
  819. /* Index is fine - add IOMMU to the array */
  820. amd_iommus[iommu->index] = iommu;
  821. /*
  822. * Copy data from ACPI table entry to the iommu struct
  823. */
  824. iommu->devid = h->devid;
  825. iommu->cap_ptr = h->cap_ptr;
  826. iommu->pci_seg = h->pci_seg;
  827. iommu->mmio_phys = h->mmio_phys;
  828. iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
  829. if (!iommu->mmio_base)
  830. return -ENOMEM;
  831. iommu->cmd_buf = alloc_command_buffer(iommu);
  832. if (!iommu->cmd_buf)
  833. return -ENOMEM;
  834. iommu->evt_buf = alloc_event_buffer(iommu);
  835. if (!iommu->evt_buf)
  836. return -ENOMEM;
  837. iommu->int_enabled = false;
  838. ret = init_iommu_from_acpi(iommu, h);
  839. if (ret)
  840. return ret;
  841. init_iommu_devices(iommu);
  842. return 0;
  843. }
  844. /*
  845. * Iterates over all IOMMU entries in the ACPI table, allocates the
  846. * IOMMU structure and initializes it with init_iommu_one()
  847. */
  848. static int __init init_iommu_all(struct acpi_table_header *table)
  849. {
  850. u8 *p = (u8 *)table, *end = (u8 *)table;
  851. struct ivhd_header *h;
  852. struct amd_iommu *iommu;
  853. int ret;
  854. end += table->length;
  855. p += IVRS_HEADER_LENGTH;
  856. while (p < end) {
  857. h = (struct ivhd_header *)p;
  858. switch (*p) {
  859. case ACPI_IVHD_TYPE:
  860. DUMP_printk("device: %02x:%02x.%01x cap: %04x "
  861. "seg: %d flags: %01x info %04x\n",
  862. PCI_BUS(h->devid), PCI_SLOT(h->devid),
  863. PCI_FUNC(h->devid), h->cap_ptr,
  864. h->pci_seg, h->flags, h->info);
  865. DUMP_printk(" mmio-addr: %016llx\n",
  866. h->mmio_phys);
  867. iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
  868. if (iommu == NULL)
  869. return -ENOMEM;
  870. ret = init_iommu_one(iommu, h);
  871. if (ret)
  872. return ret;
  873. break;
  874. default:
  875. break;
  876. }
  877. p += h->length;
  878. }
  879. WARN_ON(p != end);
  880. return 0;
  881. }
  882. static int iommu_init_pci(struct amd_iommu *iommu)
  883. {
  884. int cap_ptr = iommu->cap_ptr;
  885. u32 range, misc, low, high;
  886. iommu->dev = pci_get_bus_and_slot(PCI_BUS(iommu->devid),
  887. iommu->devid & 0xff);
  888. if (!iommu->dev)
  889. return -ENODEV;
  890. pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
  891. &iommu->cap);
  892. pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
  893. &range);
  894. pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
  895. &misc);
  896. iommu->first_device = calc_devid(MMIO_GET_BUS(range),
  897. MMIO_GET_FD(range));
  898. iommu->last_device = calc_devid(MMIO_GET_BUS(range),
  899. MMIO_GET_LD(range));
  900. if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
  901. amd_iommu_iotlb_sup = false;
  902. /* read extended feature bits */
  903. low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
  904. high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
  905. iommu->features = ((u64)high << 32) | low;
  906. if (iommu_feature(iommu, FEATURE_GT)) {
  907. int glxval;
  908. u32 pasids;
  909. u64 shift;
  910. shift = iommu->features & FEATURE_PASID_MASK;
  911. shift >>= FEATURE_PASID_SHIFT;
  912. pasids = (1 << shift);
  913. amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
  914. glxval = iommu->features & FEATURE_GLXVAL_MASK;
  915. glxval >>= FEATURE_GLXVAL_SHIFT;
  916. if (amd_iommu_max_glx_val == -1)
  917. amd_iommu_max_glx_val = glxval;
  918. else
  919. amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
  920. }
  921. if (iommu_feature(iommu, FEATURE_GT) &&
  922. iommu_feature(iommu, FEATURE_PPR)) {
  923. iommu->is_iommu_v2 = true;
  924. amd_iommu_v2_present = true;
  925. }
  926. if (iommu_feature(iommu, FEATURE_PPR)) {
  927. iommu->ppr_log = alloc_ppr_log(iommu);
  928. if (!iommu->ppr_log)
  929. return -ENOMEM;
  930. }
  931. if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
  932. amd_iommu_np_cache = true;
  933. if (is_rd890_iommu(iommu->dev)) {
  934. int i, j;
  935. iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
  936. PCI_DEVFN(0, 0));
  937. /*
  938. * Some rd890 systems may not be fully reconfigured by the
  939. * BIOS, so it's necessary for us to store this information so
  940. * it can be reprogrammed on resume
  941. */
  942. pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
  943. &iommu->stored_addr_lo);
  944. pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
  945. &iommu->stored_addr_hi);
  946. /* Low bit locks writes to configuration space */
  947. iommu->stored_addr_lo &= ~1;
  948. for (i = 0; i < 6; i++)
  949. for (j = 0; j < 0x12; j++)
  950. iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
  951. for (i = 0; i < 0x83; i++)
  952. iommu->stored_l2[i] = iommu_read_l2(iommu, i);
  953. }
  954. return pci_enable_device(iommu->dev);
  955. }
  956. static void print_iommu_info(void)
  957. {
  958. static const char * const feat_str[] = {
  959. "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
  960. "IA", "GA", "HE", "PC"
  961. };
  962. struct amd_iommu *iommu;
  963. for_each_iommu(iommu) {
  964. int i;
  965. pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
  966. dev_name(&iommu->dev->dev), iommu->cap_ptr);
  967. if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
  968. pr_info("AMD-Vi: Extended features: ");
  969. for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
  970. if (iommu_feature(iommu, (1ULL << i)))
  971. pr_cont(" %s", feat_str[i]);
  972. }
  973. }
  974. pr_cont("\n");
  975. }
  976. }
  977. static int __init amd_iommu_init_pci(void)
  978. {
  979. struct amd_iommu *iommu;
  980. int ret = 0;
  981. for_each_iommu(iommu) {
  982. ret = iommu_init_pci(iommu);
  983. if (ret)
  984. break;
  985. }
  986. ret = amd_iommu_init_devices();
  987. print_iommu_info();
  988. return ret;
  989. }
  990. /****************************************************************************
  991. *
  992. * The following functions initialize the MSI interrupts for all IOMMUs
  993. * in the system. Its a bit challenging because there could be multiple
  994. * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
  995. * pci_dev.
  996. *
  997. ****************************************************************************/
  998. static int iommu_setup_msi(struct amd_iommu *iommu)
  999. {
  1000. int r;
  1001. r = pci_enable_msi(iommu->dev);
  1002. if (r)
  1003. return r;
  1004. r = request_threaded_irq(iommu->dev->irq,
  1005. amd_iommu_int_handler,
  1006. amd_iommu_int_thread,
  1007. 0, "AMD-Vi",
  1008. iommu->dev);
  1009. if (r) {
  1010. pci_disable_msi(iommu->dev);
  1011. return r;
  1012. }
  1013. iommu->int_enabled = true;
  1014. return 0;
  1015. }
  1016. static int iommu_init_msi(struct amd_iommu *iommu)
  1017. {
  1018. int ret;
  1019. if (iommu->int_enabled)
  1020. goto enable_faults;
  1021. if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
  1022. ret = iommu_setup_msi(iommu);
  1023. else
  1024. ret = -ENODEV;
  1025. if (ret)
  1026. return ret;
  1027. enable_faults:
  1028. iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
  1029. if (iommu->ppr_log != NULL)
  1030. iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
  1031. return 0;
  1032. }
  1033. /****************************************************************************
  1034. *
  1035. * The next functions belong to the third pass of parsing the ACPI
  1036. * table. In this last pass the memory mapping requirements are
  1037. * gathered (like exclusion and unity mapping reanges).
  1038. *
  1039. ****************************************************************************/
  1040. static void __init free_unity_maps(void)
  1041. {
  1042. struct unity_map_entry *entry, *next;
  1043. list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
  1044. list_del(&entry->list);
  1045. kfree(entry);
  1046. }
  1047. }
  1048. /* called when we find an exclusion range definition in ACPI */
  1049. static int __init init_exclusion_range(struct ivmd_header *m)
  1050. {
  1051. int i;
  1052. switch (m->type) {
  1053. case ACPI_IVMD_TYPE:
  1054. set_device_exclusion_range(m->devid, m);
  1055. break;
  1056. case ACPI_IVMD_TYPE_ALL:
  1057. for (i = 0; i <= amd_iommu_last_bdf; ++i)
  1058. set_device_exclusion_range(i, m);
  1059. break;
  1060. case ACPI_IVMD_TYPE_RANGE:
  1061. for (i = m->devid; i <= m->aux; ++i)
  1062. set_device_exclusion_range(i, m);
  1063. break;
  1064. default:
  1065. break;
  1066. }
  1067. return 0;
  1068. }
  1069. /* called for unity map ACPI definition */
  1070. static int __init init_unity_map_range(struct ivmd_header *m)
  1071. {
  1072. struct unity_map_entry *e = NULL;
  1073. char *s;
  1074. e = kzalloc(sizeof(*e), GFP_KERNEL);
  1075. if (e == NULL)
  1076. return -ENOMEM;
  1077. switch (m->type) {
  1078. default:
  1079. kfree(e);
  1080. return 0;
  1081. case ACPI_IVMD_TYPE:
  1082. s = "IVMD_TYPEi\t\t\t";
  1083. e->devid_start = e->devid_end = m->devid;
  1084. break;
  1085. case ACPI_IVMD_TYPE_ALL:
  1086. s = "IVMD_TYPE_ALL\t\t";
  1087. e->devid_start = 0;
  1088. e->devid_end = amd_iommu_last_bdf;
  1089. break;
  1090. case ACPI_IVMD_TYPE_RANGE:
  1091. s = "IVMD_TYPE_RANGE\t\t";
  1092. e->devid_start = m->devid;
  1093. e->devid_end = m->aux;
  1094. break;
  1095. }
  1096. e->address_start = PAGE_ALIGN(m->range_start);
  1097. e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
  1098. e->prot = m->flags >> 1;
  1099. DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
  1100. " range_start: %016llx range_end: %016llx flags: %x\n", s,
  1101. PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),
  1102. PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),
  1103. PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
  1104. e->address_start, e->address_end, m->flags);
  1105. list_add_tail(&e->list, &amd_iommu_unity_map);
  1106. return 0;
  1107. }
  1108. /* iterates over all memory definitions we find in the ACPI table */
  1109. static int __init init_memory_definitions(struct acpi_table_header *table)
  1110. {
  1111. u8 *p = (u8 *)table, *end = (u8 *)table;
  1112. struct ivmd_header *m;
  1113. end += table->length;
  1114. p += IVRS_HEADER_LENGTH;
  1115. while (p < end) {
  1116. m = (struct ivmd_header *)p;
  1117. if (m->flags & IVMD_FLAG_EXCL_RANGE)
  1118. init_exclusion_range(m);
  1119. else if (m->flags & IVMD_FLAG_UNITY_MAP)
  1120. init_unity_map_range(m);
  1121. p += m->length;
  1122. }
  1123. return 0;
  1124. }
  1125. /*
  1126. * Init the device table to not allow DMA access for devices and
  1127. * suppress all page faults
  1128. */
  1129. static void init_device_table(void)
  1130. {
  1131. u32 devid;
  1132. for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
  1133. set_dev_entry_bit(devid, DEV_ENTRY_VALID);
  1134. set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
  1135. }
  1136. }
  1137. static void iommu_init_flags(struct amd_iommu *iommu)
  1138. {
  1139. iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
  1140. iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
  1141. iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
  1142. iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
  1143. iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
  1144. iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
  1145. iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
  1146. iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
  1147. iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
  1148. iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
  1149. iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
  1150. iommu_feature_disable(iommu, CONTROL_ISOC_EN);
  1151. /*
  1152. * make IOMMU memory accesses cache coherent
  1153. */
  1154. iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
  1155. /* Set IOTLB invalidation timeout to 1s */
  1156. iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
  1157. }
  1158. static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
  1159. {
  1160. int i, j;
  1161. u32 ioc_feature_control;
  1162. struct pci_dev *pdev = iommu->root_pdev;
  1163. /* RD890 BIOSes may not have completely reconfigured the iommu */
  1164. if (!is_rd890_iommu(iommu->dev) || !pdev)
  1165. return;
  1166. /*
  1167. * First, we need to ensure that the iommu is enabled. This is
  1168. * controlled by a register in the northbridge
  1169. */
  1170. /* Select Northbridge indirect register 0x75 and enable writing */
  1171. pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
  1172. pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
  1173. /* Enable the iommu */
  1174. if (!(ioc_feature_control & 0x1))
  1175. pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
  1176. /* Restore the iommu BAR */
  1177. pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
  1178. iommu->stored_addr_lo);
  1179. pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
  1180. iommu->stored_addr_hi);
  1181. /* Restore the l1 indirect regs for each of the 6 l1s */
  1182. for (i = 0; i < 6; i++)
  1183. for (j = 0; j < 0x12; j++)
  1184. iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
  1185. /* Restore the l2 indirect regs */
  1186. for (i = 0; i < 0x83; i++)
  1187. iommu_write_l2(iommu, i, iommu->stored_l2[i]);
  1188. /* Lock PCI setup registers */
  1189. pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
  1190. iommu->stored_addr_lo | 1);
  1191. }
  1192. /*
  1193. * This function finally enables all IOMMUs found in the system after
  1194. * they have been initialized
  1195. */
  1196. static void early_enable_iommus(void)
  1197. {
  1198. struct amd_iommu *iommu;
  1199. for_each_iommu(iommu) {
  1200. iommu_disable(iommu);
  1201. iommu_init_flags(iommu);
  1202. iommu_set_device_table(iommu);
  1203. iommu_enable_command_buffer(iommu);
  1204. iommu_enable_event_buffer(iommu);
  1205. iommu_set_exclusion_range(iommu);
  1206. iommu_enable(iommu);
  1207. iommu_flush_all_caches(iommu);
  1208. }
  1209. }
  1210. static void enable_iommus_v2(void)
  1211. {
  1212. struct amd_iommu *iommu;
  1213. for_each_iommu(iommu) {
  1214. iommu_enable_ppr_log(iommu);
  1215. iommu_enable_gt(iommu);
  1216. }
  1217. }
  1218. static void enable_iommus(void)
  1219. {
  1220. early_enable_iommus();
  1221. enable_iommus_v2();
  1222. }
  1223. static void disable_iommus(void)
  1224. {
  1225. struct amd_iommu *iommu;
  1226. for_each_iommu(iommu)
  1227. iommu_disable(iommu);
  1228. }
  1229. /*
  1230. * Suspend/Resume support
  1231. * disable suspend until real resume implemented
  1232. */
  1233. static void amd_iommu_resume(void)
  1234. {
  1235. struct amd_iommu *iommu;
  1236. for_each_iommu(iommu)
  1237. iommu_apply_resume_quirks(iommu);
  1238. /* re-load the hardware */
  1239. enable_iommus();
  1240. amd_iommu_enable_interrupts();
  1241. }
  1242. static int amd_iommu_suspend(void)
  1243. {
  1244. /* disable IOMMUs to go out of the way for BIOS */
  1245. disable_iommus();
  1246. return 0;
  1247. }
  1248. static struct syscore_ops amd_iommu_syscore_ops = {
  1249. .suspend = amd_iommu_suspend,
  1250. .resume = amd_iommu_resume,
  1251. };
  1252. static void __init free_on_init_error(void)
  1253. {
  1254. free_pages((unsigned long)irq_lookup_table,
  1255. get_order(rlookup_table_size));
  1256. if (amd_iommu_irq_cache) {
  1257. kmem_cache_destroy(amd_iommu_irq_cache);
  1258. amd_iommu_irq_cache = NULL;
  1259. }
  1260. amd_iommu_uninit_devices();
  1261. free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
  1262. get_order(MAX_DOMAIN_ID/8));
  1263. free_pages((unsigned long)amd_iommu_rlookup_table,
  1264. get_order(rlookup_table_size));
  1265. free_pages((unsigned long)amd_iommu_alias_table,
  1266. get_order(alias_table_size));
  1267. free_pages((unsigned long)amd_iommu_dev_table,
  1268. get_order(dev_table_size));
  1269. free_iommu_all();
  1270. free_unity_maps();
  1271. #ifdef CONFIG_GART_IOMMU
  1272. /*
  1273. * We failed to initialize the AMD IOMMU - try fallback to GART
  1274. * if possible.
  1275. */
  1276. gart_iommu_init();
  1277. #endif
  1278. }
  1279. /*
  1280. * This is the hardware init function for AMD IOMMU in the system.
  1281. * This function is called either from amd_iommu_init or from the interrupt
  1282. * remapping setup code.
  1283. *
  1284. * This function basically parses the ACPI table for AMD IOMMU (IVRS)
  1285. * three times:
  1286. *
  1287. * 1 pass) Find the highest PCI device id the driver has to handle.
  1288. * Upon this information the size of the data structures is
  1289. * determined that needs to be allocated.
  1290. *
  1291. * 2 pass) Initialize the data structures just allocated with the
  1292. * information in the ACPI table about available AMD IOMMUs
  1293. * in the system. It also maps the PCI devices in the
  1294. * system to specific IOMMUs
  1295. *
  1296. * 3 pass) After the basic data structures are allocated and
  1297. * initialized we update them with information about memory
  1298. * remapping requirements parsed out of the ACPI table in
  1299. * this last pass.
  1300. *
  1301. * After everything is set up the IOMMUs are enabled and the necessary
  1302. * hotplug and suspend notifiers are registered.
  1303. */
  1304. static int __init early_amd_iommu_init(void)
  1305. {
  1306. struct acpi_table_header *ivrs_base;
  1307. acpi_size ivrs_size;
  1308. acpi_status status;
  1309. int i, ret = 0;
  1310. if (!amd_iommu_detected)
  1311. return -ENODEV;
  1312. status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
  1313. if (status == AE_NOT_FOUND)
  1314. return -ENODEV;
  1315. else if (ACPI_FAILURE(status)) {
  1316. const char *err = acpi_format_exception(status);
  1317. pr_err("AMD-Vi: IVRS table error: %s\n", err);
  1318. return -EINVAL;
  1319. }
  1320. /*
  1321. * First parse ACPI tables to find the largest Bus/Dev/Func
  1322. * we need to handle. Upon this information the shared data
  1323. * structures for the IOMMUs in the system will be allocated
  1324. */
  1325. ret = find_last_devid_acpi(ivrs_base);
  1326. if (ret)
  1327. goto out;
  1328. dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
  1329. alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
  1330. rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
  1331. /* Device table - directly used by all IOMMUs */
  1332. ret = -ENOMEM;
  1333. amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  1334. get_order(dev_table_size));
  1335. if (amd_iommu_dev_table == NULL)
  1336. goto out;
  1337. /*
  1338. * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
  1339. * IOMMU see for that device
  1340. */
  1341. amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
  1342. get_order(alias_table_size));
  1343. if (amd_iommu_alias_table == NULL)
  1344. goto out;
  1345. /* IOMMU rlookup table - find the IOMMU for a specific device */
  1346. amd_iommu_rlookup_table = (void *)__get_free_pages(
  1347. GFP_KERNEL | __GFP_ZERO,
  1348. get_order(rlookup_table_size));
  1349. if (amd_iommu_rlookup_table == NULL)
  1350. goto out;
  1351. amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
  1352. GFP_KERNEL | __GFP_ZERO,
  1353. get_order(MAX_DOMAIN_ID/8));
  1354. if (amd_iommu_pd_alloc_bitmap == NULL)
  1355. goto out;
  1356. /* init the device table */
  1357. init_device_table();
  1358. /*
  1359. * let all alias entries point to itself
  1360. */
  1361. for (i = 0; i <= amd_iommu_last_bdf; ++i)
  1362. amd_iommu_alias_table[i] = i;
  1363. /*
  1364. * never allocate domain 0 because its used as the non-allocated and
  1365. * error value placeholder
  1366. */
  1367. amd_iommu_pd_alloc_bitmap[0] = 1;
  1368. spin_lock_init(&amd_iommu_pd_lock);
  1369. /*
  1370. * now the data structures are allocated and basically initialized
  1371. * start the real acpi table scan
  1372. */
  1373. ret = init_iommu_all(ivrs_base);
  1374. if (ret)
  1375. goto out;
  1376. if (amd_iommu_irq_remap) {
  1377. /*
  1378. * Interrupt remapping enabled, create kmem_cache for the
  1379. * remapping tables.
  1380. */
  1381. amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
  1382. MAX_IRQS_PER_TABLE * sizeof(u32),
  1383. IRQ_TABLE_ALIGNMENT,
  1384. 0, NULL);
  1385. if (!amd_iommu_irq_cache)
  1386. goto out;
  1387. irq_lookup_table = (void *)__get_free_pages(
  1388. GFP_KERNEL | __GFP_ZERO,
  1389. get_order(rlookup_table_size));
  1390. if (!irq_lookup_table)
  1391. goto out;
  1392. }
  1393. ret = init_memory_definitions(ivrs_base);
  1394. if (ret)
  1395. goto out;
  1396. out:
  1397. /* Don't leak any ACPI memory */
  1398. early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
  1399. ivrs_base = NULL;
  1400. return ret;
  1401. }
  1402. static int amd_iommu_enable_interrupts(void)
  1403. {
  1404. struct amd_iommu *iommu;
  1405. int ret = 0;
  1406. for_each_iommu(iommu) {
  1407. ret = iommu_init_msi(iommu);
  1408. if (ret)
  1409. goto out;
  1410. }
  1411. out:
  1412. return ret;
  1413. }
  1414. static bool detect_ivrs(void)
  1415. {
  1416. struct acpi_table_header *ivrs_base;
  1417. acpi_size ivrs_size;
  1418. acpi_status status;
  1419. status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
  1420. if (status == AE_NOT_FOUND)
  1421. return false;
  1422. else if (ACPI_FAILURE(status)) {
  1423. const char *err = acpi_format_exception(status);
  1424. pr_err("AMD-Vi: IVRS table error: %s\n", err);
  1425. return false;
  1426. }
  1427. early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
  1428. /* Make sure ACS will be enabled during PCI probe */
  1429. pci_request_acs();
  1430. if (!disable_irq_remap)
  1431. amd_iommu_irq_remap = true;
  1432. return true;
  1433. }
  1434. static int amd_iommu_init_dma(void)
  1435. {
  1436. int ret;
  1437. if (iommu_pass_through)
  1438. ret = amd_iommu_init_passthrough();
  1439. else
  1440. ret = amd_iommu_init_dma_ops();
  1441. if (ret)
  1442. return ret;
  1443. amd_iommu_init_api();
  1444. amd_iommu_init_notifier();
  1445. return 0;
  1446. }
  1447. /****************************************************************************
  1448. *
  1449. * AMD IOMMU Initialization State Machine
  1450. *
  1451. ****************************************************************************/
  1452. static int __init state_next(void)
  1453. {
  1454. int ret = 0;
  1455. switch (init_state) {
  1456. case IOMMU_START_STATE:
  1457. if (!detect_ivrs()) {
  1458. init_state = IOMMU_NOT_FOUND;
  1459. ret = -ENODEV;
  1460. } else {
  1461. init_state = IOMMU_IVRS_DETECTED;
  1462. }
  1463. break;
  1464. case IOMMU_IVRS_DETECTED:
  1465. ret = early_amd_iommu_init();
  1466. init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
  1467. break;
  1468. case IOMMU_ACPI_FINISHED:
  1469. early_enable_iommus();
  1470. register_syscore_ops(&amd_iommu_syscore_ops);
  1471. x86_platform.iommu_shutdown = disable_iommus;
  1472. init_state = IOMMU_ENABLED;
  1473. break;
  1474. case IOMMU_ENABLED:
  1475. ret = amd_iommu_init_pci();
  1476. init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
  1477. enable_iommus_v2();
  1478. break;
  1479. case IOMMU_PCI_INIT:
  1480. ret = amd_iommu_enable_interrupts();
  1481. init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
  1482. break;
  1483. case IOMMU_INTERRUPTS_EN:
  1484. ret = amd_iommu_init_dma();
  1485. init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
  1486. break;
  1487. case IOMMU_DMA_OPS:
  1488. init_state = IOMMU_INITIALIZED;
  1489. break;
  1490. case IOMMU_INITIALIZED:
  1491. /* Nothing to do */
  1492. break;
  1493. case IOMMU_NOT_FOUND:
  1494. case IOMMU_INIT_ERROR:
  1495. /* Error states => do nothing */
  1496. ret = -EINVAL;
  1497. break;
  1498. default:
  1499. /* Unknown state */
  1500. BUG();
  1501. }
  1502. return ret;
  1503. }
  1504. static int __init iommu_go_to_state(enum iommu_init_state state)
  1505. {
  1506. int ret = 0;
  1507. while (init_state != state) {
  1508. ret = state_next();
  1509. if (init_state == IOMMU_NOT_FOUND ||
  1510. init_state == IOMMU_INIT_ERROR)
  1511. break;
  1512. }
  1513. return ret;
  1514. }
  1515. /*
  1516. * This is the core init function for AMD IOMMU hardware in the system.
  1517. * This function is called from the generic x86 DMA layer initialization
  1518. * code.
  1519. */
  1520. static int __init amd_iommu_init(void)
  1521. {
  1522. int ret;
  1523. ret = iommu_go_to_state(IOMMU_INITIALIZED);
  1524. if (ret) {
  1525. disable_iommus();
  1526. free_on_init_error();
  1527. }
  1528. return ret;
  1529. }
  1530. /****************************************************************************
  1531. *
  1532. * Early detect code. This code runs at IOMMU detection time in the DMA
  1533. * layer. It just looks if there is an IVRS ACPI table to detect AMD
  1534. * IOMMUs
  1535. *
  1536. ****************************************************************************/
  1537. int __init amd_iommu_detect(void)
  1538. {
  1539. int ret;
  1540. if (no_iommu || (iommu_detected && !gart_iommu_aperture))
  1541. return -ENODEV;
  1542. if (amd_iommu_disabled)
  1543. return -ENODEV;
  1544. ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
  1545. if (ret)
  1546. return ret;
  1547. amd_iommu_detected = true;
  1548. iommu_detected = 1;
  1549. x86_init.iommu.iommu_init = amd_iommu_init;
  1550. return 0;
  1551. }
  1552. /****************************************************************************
  1553. *
  1554. * Parsing functions for the AMD IOMMU specific kernel command line
  1555. * options.
  1556. *
  1557. ****************************************************************************/
  1558. static int __init parse_amd_iommu_dump(char *str)
  1559. {
  1560. amd_iommu_dump = true;
  1561. return 1;
  1562. }
  1563. static int __init parse_amd_iommu_options(char *str)
  1564. {
  1565. for (; *str; ++str) {
  1566. if (strncmp(str, "fullflush", 9) == 0)
  1567. amd_iommu_unmap_flush = true;
  1568. if (strncmp(str, "off", 3) == 0)
  1569. amd_iommu_disabled = true;
  1570. if (strncmp(str, "force_isolation", 15) == 0)
  1571. amd_iommu_force_isolation = true;
  1572. }
  1573. return 1;
  1574. }
  1575. __setup("amd_iommu_dump", parse_amd_iommu_dump);
  1576. __setup("amd_iommu=", parse_amd_iommu_options);
  1577. IOMMU_INIT_FINISH(amd_iommu_detect,
  1578. gart_iommu_hole_init,
  1579. NULL,
  1580. NULL);
  1581. bool amd_iommu_v2_supported(void)
  1582. {
  1583. return amd_iommu_v2_present;
  1584. }
  1585. EXPORT_SYMBOL(amd_iommu_v2_supported);