amd_iommu.c 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681
  1. /*
  2. * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  3. * Author: Joerg Roedel <joerg.roedel@amd.com>
  4. * Leo Duran <leo.duran@amd.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/ratelimit.h>
  20. #include <linux/pci.h>
  21. #include <linux/pci-ats.h>
  22. #include <linux/bitmap.h>
  23. #include <linux/slab.h>
  24. #include <linux/debugfs.h>
  25. #include <linux/scatterlist.h>
  26. #include <linux/dma-mapping.h>
  27. #include <linux/iommu-helper.h>
  28. #include <linux/iommu.h>
  29. #include <linux/delay.h>
  30. #include <linux/amd-iommu.h>
  31. #include <linux/notifier.h>
  32. #include <linux/export.h>
  33. #include <asm/msidef.h>
  34. #include <asm/proto.h>
  35. #include <asm/iommu.h>
  36. #include <asm/gart.h>
  37. #include <asm/dma.h>
  38. #include "amd_iommu_proto.h"
  39. #include "amd_iommu_types.h"
  40. #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  41. #define LOOP_TIMEOUT 100000
  42. /*
  43. * This bitmap is used to advertise the page sizes our hardware support
  44. * to the IOMMU core, which will then use this information to split
  45. * physically contiguous memory regions it is mapping into page sizes
  46. * that we support.
  47. *
  48. * Traditionally the IOMMU core just handed us the mappings directly,
  49. * after making sure the size is an order of a 4KiB page and that the
  50. * mapping has natural alignment.
  51. *
  52. * To retain this behavior, we currently advertise that we support
  53. * all page sizes that are an order of 4KiB.
  54. *
  55. * If at some point we'd like to utilize the IOMMU core's new behavior,
  56. * we could change this to advertise the real page sizes we support.
  57. */
  58. #define AMD_IOMMU_PGSIZES (~0xFFFUL)
  59. static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  60. /* A list of preallocated protection domains */
  61. static LIST_HEAD(iommu_pd_list);
  62. static DEFINE_SPINLOCK(iommu_pd_list_lock);
  63. /* List of all available dev_data structures */
  64. static LIST_HEAD(dev_data_list);
  65. static DEFINE_SPINLOCK(dev_data_list_lock);
  66. /*
  67. * Domain for untranslated devices - only allocated
  68. * if iommu=pt passed on kernel cmd line.
  69. */
  70. static struct protection_domain *pt_domain;
  71. static struct iommu_ops amd_iommu_ops;
  72. static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
  73. int amd_iommu_max_glx_val = -1;
  74. static struct dma_map_ops amd_iommu_dma_ops;
  75. /*
  76. * general struct to manage commands send to an IOMMU
  77. */
  78. struct iommu_cmd {
  79. u32 data[4];
  80. };
  81. static void update_domain(struct protection_domain *domain);
  82. static int __init alloc_passthrough_domain(void);
  83. /****************************************************************************
  84. *
  85. * Helper functions
  86. *
  87. ****************************************************************************/
  88. static struct iommu_dev_data *alloc_dev_data(u16 devid)
  89. {
  90. struct iommu_dev_data *dev_data;
  91. unsigned long flags;
  92. dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
  93. if (!dev_data)
  94. return NULL;
  95. dev_data->devid = devid;
  96. atomic_set(&dev_data->bind, 0);
  97. spin_lock_irqsave(&dev_data_list_lock, flags);
  98. list_add_tail(&dev_data->dev_data_list, &dev_data_list);
  99. spin_unlock_irqrestore(&dev_data_list_lock, flags);
  100. return dev_data;
  101. }
  102. static void free_dev_data(struct iommu_dev_data *dev_data)
  103. {
  104. unsigned long flags;
  105. spin_lock_irqsave(&dev_data_list_lock, flags);
  106. list_del(&dev_data->dev_data_list);
  107. spin_unlock_irqrestore(&dev_data_list_lock, flags);
  108. kfree(dev_data);
  109. }
  110. static struct iommu_dev_data *search_dev_data(u16 devid)
  111. {
  112. struct iommu_dev_data *dev_data;
  113. unsigned long flags;
  114. spin_lock_irqsave(&dev_data_list_lock, flags);
  115. list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
  116. if (dev_data->devid == devid)
  117. goto out_unlock;
  118. }
  119. dev_data = NULL;
  120. out_unlock:
  121. spin_unlock_irqrestore(&dev_data_list_lock, flags);
  122. return dev_data;
  123. }
  124. static struct iommu_dev_data *find_dev_data(u16 devid)
  125. {
  126. struct iommu_dev_data *dev_data;
  127. dev_data = search_dev_data(devid);
  128. if (dev_data == NULL)
  129. dev_data = alloc_dev_data(devid);
  130. return dev_data;
  131. }
  132. static inline u16 get_device_id(struct device *dev)
  133. {
  134. struct pci_dev *pdev = to_pci_dev(dev);
  135. return calc_devid(pdev->bus->number, pdev->devfn);
  136. }
  137. static struct iommu_dev_data *get_dev_data(struct device *dev)
  138. {
  139. return dev->archdata.iommu;
  140. }
  141. static bool pci_iommuv2_capable(struct pci_dev *pdev)
  142. {
  143. static const int caps[] = {
  144. PCI_EXT_CAP_ID_ATS,
  145. PCI_EXT_CAP_ID_PRI,
  146. PCI_EXT_CAP_ID_PASID,
  147. };
  148. int i, pos;
  149. for (i = 0; i < 3; ++i) {
  150. pos = pci_find_ext_capability(pdev, caps[i]);
  151. if (pos == 0)
  152. return false;
  153. }
  154. return true;
  155. }
  156. static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
  157. {
  158. struct iommu_dev_data *dev_data;
  159. dev_data = get_dev_data(&pdev->dev);
  160. return dev_data->errata & (1 << erratum) ? true : false;
  161. }
  162. /*
  163. * In this function the list of preallocated protection domains is traversed to
  164. * find the domain for a specific device
  165. */
  166. static struct dma_ops_domain *find_protection_domain(u16 devid)
  167. {
  168. struct dma_ops_domain *entry, *ret = NULL;
  169. unsigned long flags;
  170. u16 alias = amd_iommu_alias_table[devid];
  171. if (list_empty(&iommu_pd_list))
  172. return NULL;
  173. spin_lock_irqsave(&iommu_pd_list_lock, flags);
  174. list_for_each_entry(entry, &iommu_pd_list, list) {
  175. if (entry->target_dev == devid ||
  176. entry->target_dev == alias) {
  177. ret = entry;
  178. break;
  179. }
  180. }
  181. spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  182. return ret;
  183. }
  184. /*
  185. * This function checks if the driver got a valid device from the caller to
  186. * avoid dereferencing invalid pointers.
  187. */
  188. static bool check_device(struct device *dev)
  189. {
  190. u16 devid;
  191. if (!dev || !dev->dma_mask)
  192. return false;
  193. /* No device or no PCI device */
  194. if (dev->bus != &pci_bus_type)
  195. return false;
  196. devid = get_device_id(dev);
  197. /* Out of our scope? */
  198. if (devid > amd_iommu_last_bdf)
  199. return false;
  200. if (amd_iommu_rlookup_table[devid] == NULL)
  201. return false;
  202. return true;
  203. }
  204. static int iommu_init_device(struct device *dev)
  205. {
  206. struct pci_dev *pdev = to_pci_dev(dev);
  207. struct iommu_dev_data *dev_data;
  208. u16 alias;
  209. if (dev->archdata.iommu)
  210. return 0;
  211. dev_data = find_dev_data(get_device_id(dev));
  212. if (!dev_data)
  213. return -ENOMEM;
  214. alias = amd_iommu_alias_table[dev_data->devid];
  215. if (alias != dev_data->devid) {
  216. struct iommu_dev_data *alias_data;
  217. alias_data = find_dev_data(alias);
  218. if (alias_data == NULL) {
  219. pr_err("AMD-Vi: Warning: Unhandled device %s\n",
  220. dev_name(dev));
  221. free_dev_data(dev_data);
  222. return -ENOTSUPP;
  223. }
  224. dev_data->alias_data = alias_data;
  225. }
  226. if (pci_iommuv2_capable(pdev)) {
  227. struct amd_iommu *iommu;
  228. iommu = amd_iommu_rlookup_table[dev_data->devid];
  229. dev_data->iommu_v2 = iommu->is_iommu_v2;
  230. }
  231. dev->archdata.iommu = dev_data;
  232. return 0;
  233. }
  234. static void iommu_ignore_device(struct device *dev)
  235. {
  236. u16 devid, alias;
  237. devid = get_device_id(dev);
  238. alias = amd_iommu_alias_table[devid];
  239. memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
  240. memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
  241. amd_iommu_rlookup_table[devid] = NULL;
  242. amd_iommu_rlookup_table[alias] = NULL;
  243. }
  244. static void iommu_uninit_device(struct device *dev)
  245. {
  246. /*
  247. * Nothing to do here - we keep dev_data around for unplugged devices
  248. * and reuse it when the device is re-plugged - not doing so would
  249. * introduce a ton of races.
  250. */
  251. }
  252. void __init amd_iommu_uninit_devices(void)
  253. {
  254. struct iommu_dev_data *dev_data, *n;
  255. struct pci_dev *pdev = NULL;
  256. for_each_pci_dev(pdev) {
  257. if (!check_device(&pdev->dev))
  258. continue;
  259. iommu_uninit_device(&pdev->dev);
  260. }
  261. /* Free all of our dev_data structures */
  262. list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
  263. free_dev_data(dev_data);
  264. }
  265. int __init amd_iommu_init_devices(void)
  266. {
  267. struct pci_dev *pdev = NULL;
  268. int ret = 0;
  269. for_each_pci_dev(pdev) {
  270. if (!check_device(&pdev->dev))
  271. continue;
  272. ret = iommu_init_device(&pdev->dev);
  273. if (ret == -ENOTSUPP)
  274. iommu_ignore_device(&pdev->dev);
  275. else if (ret)
  276. goto out_free;
  277. }
  278. return 0;
  279. out_free:
  280. amd_iommu_uninit_devices();
  281. return ret;
  282. }
  283. #ifdef CONFIG_AMD_IOMMU_STATS
  284. /*
  285. * Initialization code for statistics collection
  286. */
  287. DECLARE_STATS_COUNTER(compl_wait);
  288. DECLARE_STATS_COUNTER(cnt_map_single);
  289. DECLARE_STATS_COUNTER(cnt_unmap_single);
  290. DECLARE_STATS_COUNTER(cnt_map_sg);
  291. DECLARE_STATS_COUNTER(cnt_unmap_sg);
  292. DECLARE_STATS_COUNTER(cnt_alloc_coherent);
  293. DECLARE_STATS_COUNTER(cnt_free_coherent);
  294. DECLARE_STATS_COUNTER(cross_page);
  295. DECLARE_STATS_COUNTER(domain_flush_single);
  296. DECLARE_STATS_COUNTER(domain_flush_all);
  297. DECLARE_STATS_COUNTER(alloced_io_mem);
  298. DECLARE_STATS_COUNTER(total_map_requests);
  299. DECLARE_STATS_COUNTER(complete_ppr);
  300. DECLARE_STATS_COUNTER(invalidate_iotlb);
  301. DECLARE_STATS_COUNTER(invalidate_iotlb_all);
  302. DECLARE_STATS_COUNTER(pri_requests);
  303. static struct dentry *stats_dir;
  304. static struct dentry *de_fflush;
  305. static void amd_iommu_stats_add(struct __iommu_counter *cnt)
  306. {
  307. if (stats_dir == NULL)
  308. return;
  309. cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
  310. &cnt->value);
  311. }
  312. static void amd_iommu_stats_init(void)
  313. {
  314. stats_dir = debugfs_create_dir("amd-iommu", NULL);
  315. if (stats_dir == NULL)
  316. return;
  317. de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
  318. &amd_iommu_unmap_flush);
  319. amd_iommu_stats_add(&compl_wait);
  320. amd_iommu_stats_add(&cnt_map_single);
  321. amd_iommu_stats_add(&cnt_unmap_single);
  322. amd_iommu_stats_add(&cnt_map_sg);
  323. amd_iommu_stats_add(&cnt_unmap_sg);
  324. amd_iommu_stats_add(&cnt_alloc_coherent);
  325. amd_iommu_stats_add(&cnt_free_coherent);
  326. amd_iommu_stats_add(&cross_page);
  327. amd_iommu_stats_add(&domain_flush_single);
  328. amd_iommu_stats_add(&domain_flush_all);
  329. amd_iommu_stats_add(&alloced_io_mem);
  330. amd_iommu_stats_add(&total_map_requests);
  331. amd_iommu_stats_add(&complete_ppr);
  332. amd_iommu_stats_add(&invalidate_iotlb);
  333. amd_iommu_stats_add(&invalidate_iotlb_all);
  334. amd_iommu_stats_add(&pri_requests);
  335. }
  336. #endif
  337. /****************************************************************************
  338. *
  339. * Interrupt handling functions
  340. *
  341. ****************************************************************************/
  342. static void dump_dte_entry(u16 devid)
  343. {
  344. int i;
  345. for (i = 0; i < 4; ++i)
  346. pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
  347. amd_iommu_dev_table[devid].data[i]);
  348. }
  349. static void dump_command(unsigned long phys_addr)
  350. {
  351. struct iommu_cmd *cmd = phys_to_virt(phys_addr);
  352. int i;
  353. for (i = 0; i < 4; ++i)
  354. pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
  355. }
  356. static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
  357. {
  358. int type, devid, domid, flags;
  359. volatile u32 *event = __evt;
  360. int count = 0;
  361. u64 address;
  362. retry:
  363. type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
  364. devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
  365. domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
  366. flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
  367. address = (u64)(((u64)event[3]) << 32) | event[2];
  368. if (type == 0) {
  369. /* Did we hit the erratum? */
  370. if (++count == LOOP_TIMEOUT) {
  371. pr_err("AMD-Vi: No event written to event log\n");
  372. return;
  373. }
  374. udelay(1);
  375. goto retry;
  376. }
  377. printk(KERN_ERR "AMD-Vi: Event logged [");
  378. switch (type) {
  379. case EVENT_TYPE_ILL_DEV:
  380. printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
  381. "address=0x%016llx flags=0x%04x]\n",
  382. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  383. address, flags);
  384. dump_dte_entry(devid);
  385. break;
  386. case EVENT_TYPE_IO_FAULT:
  387. printk("IO_PAGE_FAULT device=%02x:%02x.%x "
  388. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  389. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  390. domid, address, flags);
  391. break;
  392. case EVENT_TYPE_DEV_TAB_ERR:
  393. printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  394. "address=0x%016llx flags=0x%04x]\n",
  395. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  396. address, flags);
  397. break;
  398. case EVENT_TYPE_PAGE_TAB_ERR:
  399. printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  400. "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
  401. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  402. domid, address, flags);
  403. break;
  404. case EVENT_TYPE_ILL_CMD:
  405. printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
  406. dump_command(address);
  407. break;
  408. case EVENT_TYPE_CMD_HARD_ERR:
  409. printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
  410. "flags=0x%04x]\n", address, flags);
  411. break;
  412. case EVENT_TYPE_IOTLB_INV_TO:
  413. printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
  414. "address=0x%016llx]\n",
  415. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  416. address);
  417. break;
  418. case EVENT_TYPE_INV_DEV_REQ:
  419. printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
  420. "address=0x%016llx flags=0x%04x]\n",
  421. PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  422. address, flags);
  423. break;
  424. default:
  425. printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
  426. }
  427. memset(__evt, 0, 4 * sizeof(u32));
  428. }
  429. static void iommu_poll_events(struct amd_iommu *iommu)
  430. {
  431. u32 head, tail;
  432. unsigned long flags;
  433. spin_lock_irqsave(&iommu->lock, flags);
  434. head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  435. tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
  436. while (head != tail) {
  437. iommu_print_event(iommu, iommu->evt_buf + head);
  438. head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
  439. }
  440. writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  441. spin_unlock_irqrestore(&iommu->lock, flags);
  442. }
  443. static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
  444. {
  445. struct amd_iommu_fault fault;
  446. INC_STATS_COUNTER(pri_requests);
  447. if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
  448. pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
  449. return;
  450. }
  451. fault.address = raw[1];
  452. fault.pasid = PPR_PASID(raw[0]);
  453. fault.device_id = PPR_DEVID(raw[0]);
  454. fault.tag = PPR_TAG(raw[0]);
  455. fault.flags = PPR_FLAGS(raw[0]);
  456. atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
  457. }
  458. static void iommu_poll_ppr_log(struct amd_iommu *iommu)
  459. {
  460. unsigned long flags;
  461. u32 head, tail;
  462. if (iommu->ppr_log == NULL)
  463. return;
  464. /* enable ppr interrupts again */
  465. writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
  466. spin_lock_irqsave(&iommu->lock, flags);
  467. head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  468. tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
  469. while (head != tail) {
  470. volatile u64 *raw;
  471. u64 entry[2];
  472. int i;
  473. raw = (u64 *)(iommu->ppr_log + head);
  474. /*
  475. * Hardware bug: Interrupt may arrive before the entry is
  476. * written to memory. If this happens we need to wait for the
  477. * entry to arrive.
  478. */
  479. for (i = 0; i < LOOP_TIMEOUT; ++i) {
  480. if (PPR_REQ_TYPE(raw[0]) != 0)
  481. break;
  482. udelay(1);
  483. }
  484. /* Avoid memcpy function-call overhead */
  485. entry[0] = raw[0];
  486. entry[1] = raw[1];
  487. /*
  488. * To detect the hardware bug we need to clear the entry
  489. * back to zero.
  490. */
  491. raw[0] = raw[1] = 0UL;
  492. /* Update head pointer of hardware ring-buffer */
  493. head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
  494. writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  495. /*
  496. * Release iommu->lock because ppr-handling might need to
  497. * re-aquire it
  498. */
  499. spin_unlock_irqrestore(&iommu->lock, flags);
  500. /* Handle PPR entry */
  501. iommu_handle_ppr_entry(iommu, entry);
  502. spin_lock_irqsave(&iommu->lock, flags);
  503. /* Refresh ring-buffer information */
  504. head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  505. tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
  506. }
  507. spin_unlock_irqrestore(&iommu->lock, flags);
  508. }
  509. irqreturn_t amd_iommu_int_thread(int irq, void *data)
  510. {
  511. struct amd_iommu *iommu;
  512. for_each_iommu(iommu) {
  513. iommu_poll_events(iommu);
  514. iommu_poll_ppr_log(iommu);
  515. }
  516. return IRQ_HANDLED;
  517. }
  518. irqreturn_t amd_iommu_int_handler(int irq, void *data)
  519. {
  520. return IRQ_WAKE_THREAD;
  521. }
  522. /****************************************************************************
  523. *
  524. * IOMMU command queuing functions
  525. *
  526. ****************************************************************************/
  527. static int wait_on_sem(volatile u64 *sem)
  528. {
  529. int i = 0;
  530. while (*sem == 0 && i < LOOP_TIMEOUT) {
  531. udelay(1);
  532. i += 1;
  533. }
  534. if (i == LOOP_TIMEOUT) {
  535. pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
  536. return -EIO;
  537. }
  538. return 0;
  539. }
  540. static void copy_cmd_to_buffer(struct amd_iommu *iommu,
  541. struct iommu_cmd *cmd,
  542. u32 tail)
  543. {
  544. u8 *target;
  545. target = iommu->cmd_buf + tail;
  546. tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  547. /* Copy command to buffer */
  548. memcpy(target, cmd, sizeof(*cmd));
  549. /* Tell the IOMMU about it */
  550. writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  551. }
  552. static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
  553. {
  554. WARN_ON(address & 0x7ULL);
  555. memset(cmd, 0, sizeof(*cmd));
  556. cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
  557. cmd->data[1] = upper_32_bits(__pa(address));
  558. cmd->data[2] = 1;
  559. CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
  560. }
  561. static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
  562. {
  563. memset(cmd, 0, sizeof(*cmd));
  564. cmd->data[0] = devid;
  565. CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
  566. }
  567. static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
  568. size_t size, u16 domid, int pde)
  569. {
  570. u64 pages;
  571. int s;
  572. pages = iommu_num_pages(address, size, PAGE_SIZE);
  573. s = 0;
  574. if (pages > 1) {
  575. /*
  576. * If we have to flush more than one page, flush all
  577. * TLB entries for this domain
  578. */
  579. address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  580. s = 1;
  581. }
  582. address &= PAGE_MASK;
  583. memset(cmd, 0, sizeof(*cmd));
  584. cmd->data[1] |= domid;
  585. cmd->data[2] = lower_32_bits(address);
  586. cmd->data[3] = upper_32_bits(address);
  587. CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  588. if (s) /* size bit - we flush more than one 4kb page */
  589. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  590. if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
  591. cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  592. }
  593. static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
  594. u64 address, size_t size)
  595. {
  596. u64 pages;
  597. int s;
  598. pages = iommu_num_pages(address, size, PAGE_SIZE);
  599. s = 0;
  600. if (pages > 1) {
  601. /*
  602. * If we have to flush more than one page, flush all
  603. * TLB entries for this domain
  604. */
  605. address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  606. s = 1;
  607. }
  608. address &= PAGE_MASK;
  609. memset(cmd, 0, sizeof(*cmd));
  610. cmd->data[0] = devid;
  611. cmd->data[0] |= (qdep & 0xff) << 24;
  612. cmd->data[1] = devid;
  613. cmd->data[2] = lower_32_bits(address);
  614. cmd->data[3] = upper_32_bits(address);
  615. CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
  616. if (s)
  617. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  618. }
  619. static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
  620. u64 address, bool size)
  621. {
  622. memset(cmd, 0, sizeof(*cmd));
  623. address &= ~(0xfffULL);
  624. cmd->data[0] = pasid & PASID_MASK;
  625. cmd->data[1] = domid;
  626. cmd->data[2] = lower_32_bits(address);
  627. cmd->data[3] = upper_32_bits(address);
  628. cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  629. cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
  630. if (size)
  631. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  632. CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  633. }
  634. static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
  635. int qdep, u64 address, bool size)
  636. {
  637. memset(cmd, 0, sizeof(*cmd));
  638. address &= ~(0xfffULL);
  639. cmd->data[0] = devid;
  640. cmd->data[0] |= (pasid & 0xff) << 16;
  641. cmd->data[0] |= (qdep & 0xff) << 24;
  642. cmd->data[1] = devid;
  643. cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
  644. cmd->data[2] = lower_32_bits(address);
  645. cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
  646. cmd->data[3] = upper_32_bits(address);
  647. if (size)
  648. cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  649. CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
  650. }
  651. static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
  652. int status, int tag, bool gn)
  653. {
  654. memset(cmd, 0, sizeof(*cmd));
  655. cmd->data[0] = devid;
  656. if (gn) {
  657. cmd->data[1] = pasid & PASID_MASK;
  658. cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
  659. }
  660. cmd->data[3] = tag & 0x1ff;
  661. cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
  662. CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
  663. }
  664. static void build_inv_all(struct iommu_cmd *cmd)
  665. {
  666. memset(cmd, 0, sizeof(*cmd));
  667. CMD_SET_TYPE(cmd, CMD_INV_ALL);
  668. }
  669. /*
  670. * Writes the command to the IOMMUs command buffer and informs the
  671. * hardware about the new command.
  672. */
  673. static int iommu_queue_command_sync(struct amd_iommu *iommu,
  674. struct iommu_cmd *cmd,
  675. bool sync)
  676. {
  677. u32 left, tail, head, next_tail;
  678. unsigned long flags;
  679. WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
  680. again:
  681. spin_lock_irqsave(&iommu->lock, flags);
  682. head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  683. tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  684. next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  685. left = (head - next_tail) % iommu->cmd_buf_size;
  686. if (left <= 2) {
  687. struct iommu_cmd sync_cmd;
  688. volatile u64 sem = 0;
  689. int ret;
  690. build_completion_wait(&sync_cmd, (u64)&sem);
  691. copy_cmd_to_buffer(iommu, &sync_cmd, tail);
  692. spin_unlock_irqrestore(&iommu->lock, flags);
  693. if ((ret = wait_on_sem(&sem)) != 0)
  694. return ret;
  695. goto again;
  696. }
  697. copy_cmd_to_buffer(iommu, cmd, tail);
  698. /* We need to sync now to make sure all commands are processed */
  699. iommu->need_sync = sync;
  700. spin_unlock_irqrestore(&iommu->lock, flags);
  701. return 0;
  702. }
  703. static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  704. {
  705. return iommu_queue_command_sync(iommu, cmd, true);
  706. }
  707. /*
  708. * This function queues a completion wait command into the command
  709. * buffer of an IOMMU
  710. */
  711. static int iommu_completion_wait(struct amd_iommu *iommu)
  712. {
  713. struct iommu_cmd cmd;
  714. volatile u64 sem = 0;
  715. int ret;
  716. if (!iommu->need_sync)
  717. return 0;
  718. build_completion_wait(&cmd, (u64)&sem);
  719. ret = iommu_queue_command_sync(iommu, &cmd, false);
  720. if (ret)
  721. return ret;
  722. return wait_on_sem(&sem);
  723. }
  724. static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
  725. {
  726. struct iommu_cmd cmd;
  727. build_inv_dte(&cmd, devid);
  728. return iommu_queue_command(iommu, &cmd);
  729. }
  730. static void iommu_flush_dte_all(struct amd_iommu *iommu)
  731. {
  732. u32 devid;
  733. for (devid = 0; devid <= 0xffff; ++devid)
  734. iommu_flush_dte(iommu, devid);
  735. iommu_completion_wait(iommu);
  736. }
  737. /*
  738. * This function uses heavy locking and may disable irqs for some time. But
  739. * this is no issue because it is only called during resume.
  740. */
  741. static void iommu_flush_tlb_all(struct amd_iommu *iommu)
  742. {
  743. u32 dom_id;
  744. for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
  745. struct iommu_cmd cmd;
  746. build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
  747. dom_id, 1);
  748. iommu_queue_command(iommu, &cmd);
  749. }
  750. iommu_completion_wait(iommu);
  751. }
  752. static void iommu_flush_all(struct amd_iommu *iommu)
  753. {
  754. struct iommu_cmd cmd;
  755. build_inv_all(&cmd);
  756. iommu_queue_command(iommu, &cmd);
  757. iommu_completion_wait(iommu);
  758. }
  759. void iommu_flush_all_caches(struct amd_iommu *iommu)
  760. {
  761. if (iommu_feature(iommu, FEATURE_IA)) {
  762. iommu_flush_all(iommu);
  763. } else {
  764. iommu_flush_dte_all(iommu);
  765. iommu_flush_tlb_all(iommu);
  766. }
  767. }
  768. /*
  769. * Command send function for flushing on-device TLB
  770. */
  771. static int device_flush_iotlb(struct iommu_dev_data *dev_data,
  772. u64 address, size_t size)
  773. {
  774. struct amd_iommu *iommu;
  775. struct iommu_cmd cmd;
  776. int qdep;
  777. qdep = dev_data->ats.qdep;
  778. iommu = amd_iommu_rlookup_table[dev_data->devid];
  779. build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
  780. return iommu_queue_command(iommu, &cmd);
  781. }
  782. /*
  783. * Command send function for invalidating a device table entry
  784. */
  785. static int device_flush_dte(struct iommu_dev_data *dev_data)
  786. {
  787. struct amd_iommu *iommu;
  788. int ret;
  789. iommu = amd_iommu_rlookup_table[dev_data->devid];
  790. ret = iommu_flush_dte(iommu, dev_data->devid);
  791. if (ret)
  792. return ret;
  793. if (dev_data->ats.enabled)
  794. ret = device_flush_iotlb(dev_data, 0, ~0UL);
  795. return ret;
  796. }
  797. /*
  798. * TLB invalidation function which is called from the mapping functions.
  799. * It invalidates a single PTE if the range to flush is within a single
  800. * page. Otherwise it flushes the whole TLB of the IOMMU.
  801. */
  802. static void __domain_flush_pages(struct protection_domain *domain,
  803. u64 address, size_t size, int pde)
  804. {
  805. struct iommu_dev_data *dev_data;
  806. struct iommu_cmd cmd;
  807. int ret = 0, i;
  808. build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
  809. for (i = 0; i < amd_iommus_present; ++i) {
  810. if (!domain->dev_iommu[i])
  811. continue;
  812. /*
  813. * Devices of this domain are behind this IOMMU
  814. * We need a TLB flush
  815. */
  816. ret |= iommu_queue_command(amd_iommus[i], &cmd);
  817. }
  818. list_for_each_entry(dev_data, &domain->dev_list, list) {
  819. if (!dev_data->ats.enabled)
  820. continue;
  821. ret |= device_flush_iotlb(dev_data, address, size);
  822. }
  823. WARN_ON(ret);
  824. }
  825. static void domain_flush_pages(struct protection_domain *domain,
  826. u64 address, size_t size)
  827. {
  828. __domain_flush_pages(domain, address, size, 0);
  829. }
  830. /* Flush the whole IO/TLB for a given protection domain */
  831. static void domain_flush_tlb(struct protection_domain *domain)
  832. {
  833. __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
  834. }
  835. /* Flush the whole IO/TLB for a given protection domain - including PDE */
  836. static void domain_flush_tlb_pde(struct protection_domain *domain)
  837. {
  838. __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
  839. }
  840. static void domain_flush_complete(struct protection_domain *domain)
  841. {
  842. int i;
  843. for (i = 0; i < amd_iommus_present; ++i) {
  844. if (!domain->dev_iommu[i])
  845. continue;
  846. /*
  847. * Devices of this domain are behind this IOMMU
  848. * We need to wait for completion of all commands.
  849. */
  850. iommu_completion_wait(amd_iommus[i]);
  851. }
  852. }
  853. /*
  854. * This function flushes the DTEs for all devices in domain
  855. */
  856. static void domain_flush_devices(struct protection_domain *domain)
  857. {
  858. struct iommu_dev_data *dev_data;
  859. list_for_each_entry(dev_data, &domain->dev_list, list)
  860. device_flush_dte(dev_data);
  861. }
  862. /****************************************************************************
  863. *
  864. * The functions below are used the create the page table mappings for
  865. * unity mapped regions.
  866. *
  867. ****************************************************************************/
  868. /*
  869. * This function is used to add another level to an IO page table. Adding
  870. * another level increases the size of the address space by 9 bits to a size up
  871. * to 64 bits.
  872. */
  873. static bool increase_address_space(struct protection_domain *domain,
  874. gfp_t gfp)
  875. {
  876. u64 *pte;
  877. if (domain->mode == PAGE_MODE_6_LEVEL)
  878. /* address space already 64 bit large */
  879. return false;
  880. pte = (void *)get_zeroed_page(gfp);
  881. if (!pte)
  882. return false;
  883. *pte = PM_LEVEL_PDE(domain->mode,
  884. virt_to_phys(domain->pt_root));
  885. domain->pt_root = pte;
  886. domain->mode += 1;
  887. domain->updated = true;
  888. return true;
  889. }
  890. static u64 *alloc_pte(struct protection_domain *domain,
  891. unsigned long address,
  892. unsigned long page_size,
  893. u64 **pte_page,
  894. gfp_t gfp)
  895. {
  896. int level, end_lvl;
  897. u64 *pte, *page;
  898. BUG_ON(!is_power_of_2(page_size));
  899. while (address > PM_LEVEL_SIZE(domain->mode))
  900. increase_address_space(domain, gfp);
  901. level = domain->mode - 1;
  902. pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
  903. address = PAGE_SIZE_ALIGN(address, page_size);
  904. end_lvl = PAGE_SIZE_LEVEL(page_size);
  905. while (level > end_lvl) {
  906. if (!IOMMU_PTE_PRESENT(*pte)) {
  907. page = (u64 *)get_zeroed_page(gfp);
  908. if (!page)
  909. return NULL;
  910. *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
  911. }
  912. /* No level skipping support yet */
  913. if (PM_PTE_LEVEL(*pte) != level)
  914. return NULL;
  915. level -= 1;
  916. pte = IOMMU_PTE_PAGE(*pte);
  917. if (pte_page && level == end_lvl)
  918. *pte_page = pte;
  919. pte = &pte[PM_LEVEL_INDEX(level, address)];
  920. }
  921. return pte;
  922. }
  923. /*
  924. * This function checks if there is a PTE for a given dma address. If
  925. * there is one, it returns the pointer to it.
  926. */
  927. static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
  928. {
  929. int level;
  930. u64 *pte;
  931. if (address > PM_LEVEL_SIZE(domain->mode))
  932. return NULL;
  933. level = domain->mode - 1;
  934. pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
  935. while (level > 0) {
  936. /* Not Present */
  937. if (!IOMMU_PTE_PRESENT(*pte))
  938. return NULL;
  939. /* Large PTE */
  940. if (PM_PTE_LEVEL(*pte) == 0x07) {
  941. unsigned long pte_mask, __pte;
  942. /*
  943. * If we have a series of large PTEs, make
  944. * sure to return a pointer to the first one.
  945. */
  946. pte_mask = PTE_PAGE_SIZE(*pte);
  947. pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
  948. __pte = ((unsigned long)pte) & pte_mask;
  949. return (u64 *)__pte;
  950. }
  951. /* No level skipping support yet */
  952. if (PM_PTE_LEVEL(*pte) != level)
  953. return NULL;
  954. level -= 1;
  955. /* Walk to the next level */
  956. pte = IOMMU_PTE_PAGE(*pte);
  957. pte = &pte[PM_LEVEL_INDEX(level, address)];
  958. }
  959. return pte;
  960. }
  961. /*
  962. * Generic mapping functions. It maps a physical address into a DMA
  963. * address space. It allocates the page table pages if necessary.
  964. * In the future it can be extended to a generic mapping function
  965. * supporting all features of AMD IOMMU page tables like level skipping
  966. * and full 64 bit address spaces.
  967. */
  968. static int iommu_map_page(struct protection_domain *dom,
  969. unsigned long bus_addr,
  970. unsigned long phys_addr,
  971. int prot,
  972. unsigned long page_size)
  973. {
  974. u64 __pte, *pte;
  975. int i, count;
  976. if (!(prot & IOMMU_PROT_MASK))
  977. return -EINVAL;
  978. bus_addr = PAGE_ALIGN(bus_addr);
  979. phys_addr = PAGE_ALIGN(phys_addr);
  980. count = PAGE_SIZE_PTE_COUNT(page_size);
  981. pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
  982. for (i = 0; i < count; ++i)
  983. if (IOMMU_PTE_PRESENT(pte[i]))
  984. return -EBUSY;
  985. if (page_size > PAGE_SIZE) {
  986. __pte = PAGE_SIZE_PTE(phys_addr, page_size);
  987. __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
  988. } else
  989. __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
  990. if (prot & IOMMU_PROT_IR)
  991. __pte |= IOMMU_PTE_IR;
  992. if (prot & IOMMU_PROT_IW)
  993. __pte |= IOMMU_PTE_IW;
  994. for (i = 0; i < count; ++i)
  995. pte[i] = __pte;
  996. update_domain(dom);
  997. return 0;
  998. }
  999. static unsigned long iommu_unmap_page(struct protection_domain *dom,
  1000. unsigned long bus_addr,
  1001. unsigned long page_size)
  1002. {
  1003. unsigned long long unmap_size, unmapped;
  1004. u64 *pte;
  1005. BUG_ON(!is_power_of_2(page_size));
  1006. unmapped = 0;
  1007. while (unmapped < page_size) {
  1008. pte = fetch_pte(dom, bus_addr);
  1009. if (!pte) {
  1010. /*
  1011. * No PTE for this address
  1012. * move forward in 4kb steps
  1013. */
  1014. unmap_size = PAGE_SIZE;
  1015. } else if (PM_PTE_LEVEL(*pte) == 0) {
  1016. /* 4kb PTE found for this address */
  1017. unmap_size = PAGE_SIZE;
  1018. *pte = 0ULL;
  1019. } else {
  1020. int count, i;
  1021. /* Large PTE found which maps this address */
  1022. unmap_size = PTE_PAGE_SIZE(*pte);
  1023. count = PAGE_SIZE_PTE_COUNT(unmap_size);
  1024. for (i = 0; i < count; i++)
  1025. pte[i] = 0ULL;
  1026. }
  1027. bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
  1028. unmapped += unmap_size;
  1029. }
  1030. BUG_ON(!is_power_of_2(unmapped));
  1031. return unmapped;
  1032. }
  1033. /*
  1034. * This function checks if a specific unity mapping entry is needed for
  1035. * this specific IOMMU.
  1036. */
  1037. static int iommu_for_unity_map(struct amd_iommu *iommu,
  1038. struct unity_map_entry *entry)
  1039. {
  1040. u16 bdf, i;
  1041. for (i = entry->devid_start; i <= entry->devid_end; ++i) {
  1042. bdf = amd_iommu_alias_table[i];
  1043. if (amd_iommu_rlookup_table[bdf] == iommu)
  1044. return 1;
  1045. }
  1046. return 0;
  1047. }
  1048. /*
  1049. * This function actually applies the mapping to the page table of the
  1050. * dma_ops domain.
  1051. */
  1052. static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  1053. struct unity_map_entry *e)
  1054. {
  1055. u64 addr;
  1056. int ret;
  1057. for (addr = e->address_start; addr < e->address_end;
  1058. addr += PAGE_SIZE) {
  1059. ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
  1060. PAGE_SIZE);
  1061. if (ret)
  1062. return ret;
  1063. /*
  1064. * if unity mapping is in aperture range mark the page
  1065. * as allocated in the aperture
  1066. */
  1067. if (addr < dma_dom->aperture_size)
  1068. __set_bit(addr >> PAGE_SHIFT,
  1069. dma_dom->aperture[0]->bitmap);
  1070. }
  1071. return 0;
  1072. }
  1073. /*
  1074. * Init the unity mappings for a specific IOMMU in the system
  1075. *
  1076. * Basically iterates over all unity mapping entries and applies them to
  1077. * the default domain DMA of that IOMMU if necessary.
  1078. */
  1079. static int iommu_init_unity_mappings(struct amd_iommu *iommu)
  1080. {
  1081. struct unity_map_entry *entry;
  1082. int ret;
  1083. list_for_each_entry(entry, &amd_iommu_unity_map, list) {
  1084. if (!iommu_for_unity_map(iommu, entry))
  1085. continue;
  1086. ret = dma_ops_unity_map(iommu->default_dom, entry);
  1087. if (ret)
  1088. return ret;
  1089. }
  1090. return 0;
  1091. }
  1092. /*
  1093. * Inits the unity mappings required for a specific device
  1094. */
  1095. static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  1096. u16 devid)
  1097. {
  1098. struct unity_map_entry *e;
  1099. int ret;
  1100. list_for_each_entry(e, &amd_iommu_unity_map, list) {
  1101. if (!(devid >= e->devid_start && devid <= e->devid_end))
  1102. continue;
  1103. ret = dma_ops_unity_map(dma_dom, e);
  1104. if (ret)
  1105. return ret;
  1106. }
  1107. return 0;
  1108. }
  1109. /****************************************************************************
  1110. *
  1111. * The next functions belong to the address allocator for the dma_ops
  1112. * interface functions. They work like the allocators in the other IOMMU
  1113. * drivers. Its basically a bitmap which marks the allocated pages in
  1114. * the aperture. Maybe it could be enhanced in the future to a more
  1115. * efficient allocator.
  1116. *
  1117. ****************************************************************************/
  1118. /*
  1119. * The address allocator core functions.
  1120. *
  1121. * called with domain->lock held
  1122. */
  1123. /*
  1124. * Used to reserve address ranges in the aperture (e.g. for exclusion
  1125. * ranges.
  1126. */
  1127. static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
  1128. unsigned long start_page,
  1129. unsigned int pages)
  1130. {
  1131. unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
  1132. if (start_page + pages > last_page)
  1133. pages = last_page - start_page;
  1134. for (i = start_page; i < start_page + pages; ++i) {
  1135. int index = i / APERTURE_RANGE_PAGES;
  1136. int page = i % APERTURE_RANGE_PAGES;
  1137. __set_bit(page, dom->aperture[index]->bitmap);
  1138. }
  1139. }
  1140. /*
  1141. * This function is used to add a new aperture range to an existing
  1142. * aperture in case of dma_ops domain allocation or address allocation
  1143. * failure.
  1144. */
  1145. static int alloc_new_range(struct dma_ops_domain *dma_dom,
  1146. bool populate, gfp_t gfp)
  1147. {
  1148. int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
  1149. struct amd_iommu *iommu;
  1150. unsigned long i, old_size;
  1151. #ifdef CONFIG_IOMMU_STRESS
  1152. populate = false;
  1153. #endif
  1154. if (index >= APERTURE_MAX_RANGES)
  1155. return -ENOMEM;
  1156. dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
  1157. if (!dma_dom->aperture[index])
  1158. return -ENOMEM;
  1159. dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
  1160. if (!dma_dom->aperture[index]->bitmap)
  1161. goto out_free;
  1162. dma_dom->aperture[index]->offset = dma_dom->aperture_size;
  1163. if (populate) {
  1164. unsigned long address = dma_dom->aperture_size;
  1165. int i, num_ptes = APERTURE_RANGE_PAGES / 512;
  1166. u64 *pte, *pte_page;
  1167. for (i = 0; i < num_ptes; ++i) {
  1168. pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
  1169. &pte_page, gfp);
  1170. if (!pte)
  1171. goto out_free;
  1172. dma_dom->aperture[index]->pte_pages[i] = pte_page;
  1173. address += APERTURE_RANGE_SIZE / 64;
  1174. }
  1175. }
  1176. old_size = dma_dom->aperture_size;
  1177. dma_dom->aperture_size += APERTURE_RANGE_SIZE;
  1178. /* Reserve address range used for MSI messages */
  1179. if (old_size < MSI_ADDR_BASE_LO &&
  1180. dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
  1181. unsigned long spage;
  1182. int pages;
  1183. pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
  1184. spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
  1185. dma_ops_reserve_addresses(dma_dom, spage, pages);
  1186. }
  1187. /* Initialize the exclusion range if necessary */
  1188. for_each_iommu(iommu) {
  1189. if (iommu->exclusion_start &&
  1190. iommu->exclusion_start >= dma_dom->aperture[index]->offset
  1191. && iommu->exclusion_start < dma_dom->aperture_size) {
  1192. unsigned long startpage;
  1193. int pages = iommu_num_pages(iommu->exclusion_start,
  1194. iommu->exclusion_length,
  1195. PAGE_SIZE);
  1196. startpage = iommu->exclusion_start >> PAGE_SHIFT;
  1197. dma_ops_reserve_addresses(dma_dom, startpage, pages);
  1198. }
  1199. }
  1200. /*
  1201. * Check for areas already mapped as present in the new aperture
  1202. * range and mark those pages as reserved in the allocator. Such
  1203. * mappings may already exist as a result of requested unity
  1204. * mappings for devices.
  1205. */
  1206. for (i = dma_dom->aperture[index]->offset;
  1207. i < dma_dom->aperture_size;
  1208. i += PAGE_SIZE) {
  1209. u64 *pte = fetch_pte(&dma_dom->domain, i);
  1210. if (!pte || !IOMMU_PTE_PRESENT(*pte))
  1211. continue;
  1212. dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
  1213. }
  1214. update_domain(&dma_dom->domain);
  1215. return 0;
  1216. out_free:
  1217. update_domain(&dma_dom->domain);
  1218. free_page((unsigned long)dma_dom->aperture[index]->bitmap);
  1219. kfree(dma_dom->aperture[index]);
  1220. dma_dom->aperture[index] = NULL;
  1221. return -ENOMEM;
  1222. }
  1223. static unsigned long dma_ops_area_alloc(struct device *dev,
  1224. struct dma_ops_domain *dom,
  1225. unsigned int pages,
  1226. unsigned long align_mask,
  1227. u64 dma_mask,
  1228. unsigned long start)
  1229. {
  1230. unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
  1231. int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
  1232. int i = start >> APERTURE_RANGE_SHIFT;
  1233. unsigned long boundary_size;
  1234. unsigned long address = -1;
  1235. unsigned long limit;
  1236. next_bit >>= PAGE_SHIFT;
  1237. boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
  1238. PAGE_SIZE) >> PAGE_SHIFT;
  1239. for (;i < max_index; ++i) {
  1240. unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
  1241. if (dom->aperture[i]->offset >= dma_mask)
  1242. break;
  1243. limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
  1244. dma_mask >> PAGE_SHIFT);
  1245. address = iommu_area_alloc(dom->aperture[i]->bitmap,
  1246. limit, next_bit, pages, 0,
  1247. boundary_size, align_mask);
  1248. if (address != -1) {
  1249. address = dom->aperture[i]->offset +
  1250. (address << PAGE_SHIFT);
  1251. dom->next_address = address + (pages << PAGE_SHIFT);
  1252. break;
  1253. }
  1254. next_bit = 0;
  1255. }
  1256. return address;
  1257. }
  1258. static unsigned long dma_ops_alloc_addresses(struct device *dev,
  1259. struct dma_ops_domain *dom,
  1260. unsigned int pages,
  1261. unsigned long align_mask,
  1262. u64 dma_mask)
  1263. {
  1264. unsigned long address;
  1265. #ifdef CONFIG_IOMMU_STRESS
  1266. dom->next_address = 0;
  1267. dom->need_flush = true;
  1268. #endif
  1269. address = dma_ops_area_alloc(dev, dom, pages, align_mask,
  1270. dma_mask, dom->next_address);
  1271. if (address == -1) {
  1272. dom->next_address = 0;
  1273. address = dma_ops_area_alloc(dev, dom, pages, align_mask,
  1274. dma_mask, 0);
  1275. dom->need_flush = true;
  1276. }
  1277. if (unlikely(address == -1))
  1278. address = DMA_ERROR_CODE;
  1279. WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
  1280. return address;
  1281. }
  1282. /*
  1283. * The address free function.
  1284. *
  1285. * called with domain->lock held
  1286. */
  1287. static void dma_ops_free_addresses(struct dma_ops_domain *dom,
  1288. unsigned long address,
  1289. unsigned int pages)
  1290. {
  1291. unsigned i = address >> APERTURE_RANGE_SHIFT;
  1292. struct aperture_range *range = dom->aperture[i];
  1293. BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
  1294. #ifdef CONFIG_IOMMU_STRESS
  1295. if (i < 4)
  1296. return;
  1297. #endif
  1298. if (address >= dom->next_address)
  1299. dom->need_flush = true;
  1300. address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
  1301. bitmap_clear(range->bitmap, address, pages);
  1302. }
  1303. /****************************************************************************
  1304. *
  1305. * The next functions belong to the domain allocation. A domain is
  1306. * allocated for every IOMMU as the default domain. If device isolation
  1307. * is enabled, every device get its own domain. The most important thing
  1308. * about domains is the page table mapping the DMA address space they
  1309. * contain.
  1310. *
  1311. ****************************************************************************/
  1312. /*
  1313. * This function adds a protection domain to the global protection domain list
  1314. */
  1315. static void add_domain_to_list(struct protection_domain *domain)
  1316. {
  1317. unsigned long flags;
  1318. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  1319. list_add(&domain->list, &amd_iommu_pd_list);
  1320. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  1321. }
  1322. /*
  1323. * This function removes a protection domain to the global
  1324. * protection domain list
  1325. */
  1326. static void del_domain_from_list(struct protection_domain *domain)
  1327. {
  1328. unsigned long flags;
  1329. spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  1330. list_del(&domain->list);
  1331. spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  1332. }
  1333. static u16 domain_id_alloc(void)
  1334. {
  1335. unsigned long flags;
  1336. int id;
  1337. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1338. id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
  1339. BUG_ON(id == 0);
  1340. if (id > 0 && id < MAX_DOMAIN_ID)
  1341. __set_bit(id, amd_iommu_pd_alloc_bitmap);
  1342. else
  1343. id = 0;
  1344. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1345. return id;
  1346. }
  1347. static void domain_id_free(int id)
  1348. {
  1349. unsigned long flags;
  1350. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1351. if (id > 0 && id < MAX_DOMAIN_ID)
  1352. __clear_bit(id, amd_iommu_pd_alloc_bitmap);
  1353. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1354. }
  1355. static void free_pagetable(struct protection_domain *domain)
  1356. {
  1357. int i, j;
  1358. u64 *p1, *p2, *p3;
  1359. p1 = domain->pt_root;
  1360. if (!p1)
  1361. return;
  1362. for (i = 0; i < 512; ++i) {
  1363. if (!IOMMU_PTE_PRESENT(p1[i]))
  1364. continue;
  1365. p2 = IOMMU_PTE_PAGE(p1[i]);
  1366. for (j = 0; j < 512; ++j) {
  1367. if (!IOMMU_PTE_PRESENT(p2[j]))
  1368. continue;
  1369. p3 = IOMMU_PTE_PAGE(p2[j]);
  1370. free_page((unsigned long)p3);
  1371. }
  1372. free_page((unsigned long)p2);
  1373. }
  1374. free_page((unsigned long)p1);
  1375. domain->pt_root = NULL;
  1376. }
  1377. static void free_gcr3_tbl_level1(u64 *tbl)
  1378. {
  1379. u64 *ptr;
  1380. int i;
  1381. for (i = 0; i < 512; ++i) {
  1382. if (!(tbl[i] & GCR3_VALID))
  1383. continue;
  1384. ptr = __va(tbl[i] & PAGE_MASK);
  1385. free_page((unsigned long)ptr);
  1386. }
  1387. }
  1388. static void free_gcr3_tbl_level2(u64 *tbl)
  1389. {
  1390. u64 *ptr;
  1391. int i;
  1392. for (i = 0; i < 512; ++i) {
  1393. if (!(tbl[i] & GCR3_VALID))
  1394. continue;
  1395. ptr = __va(tbl[i] & PAGE_MASK);
  1396. free_gcr3_tbl_level1(ptr);
  1397. }
  1398. }
  1399. static void free_gcr3_table(struct protection_domain *domain)
  1400. {
  1401. if (domain->glx == 2)
  1402. free_gcr3_tbl_level2(domain->gcr3_tbl);
  1403. else if (domain->glx == 1)
  1404. free_gcr3_tbl_level1(domain->gcr3_tbl);
  1405. else if (domain->glx != 0)
  1406. BUG();
  1407. free_page((unsigned long)domain->gcr3_tbl);
  1408. }
  1409. /*
  1410. * Free a domain, only used if something went wrong in the
  1411. * allocation path and we need to free an already allocated page table
  1412. */
  1413. static void dma_ops_domain_free(struct dma_ops_domain *dom)
  1414. {
  1415. int i;
  1416. if (!dom)
  1417. return;
  1418. del_domain_from_list(&dom->domain);
  1419. free_pagetable(&dom->domain);
  1420. for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
  1421. if (!dom->aperture[i])
  1422. continue;
  1423. free_page((unsigned long)dom->aperture[i]->bitmap);
  1424. kfree(dom->aperture[i]);
  1425. }
  1426. kfree(dom);
  1427. }
  1428. /*
  1429. * Allocates a new protection domain usable for the dma_ops functions.
  1430. * It also initializes the page table and the address allocator data
  1431. * structures required for the dma_ops interface
  1432. */
  1433. static struct dma_ops_domain *dma_ops_domain_alloc(void)
  1434. {
  1435. struct dma_ops_domain *dma_dom;
  1436. dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
  1437. if (!dma_dom)
  1438. return NULL;
  1439. spin_lock_init(&dma_dom->domain.lock);
  1440. dma_dom->domain.id = domain_id_alloc();
  1441. if (dma_dom->domain.id == 0)
  1442. goto free_dma_dom;
  1443. INIT_LIST_HEAD(&dma_dom->domain.dev_list);
  1444. dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
  1445. dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
  1446. dma_dom->domain.flags = PD_DMA_OPS_MASK;
  1447. dma_dom->domain.priv = dma_dom;
  1448. if (!dma_dom->domain.pt_root)
  1449. goto free_dma_dom;
  1450. dma_dom->need_flush = false;
  1451. dma_dom->target_dev = 0xffff;
  1452. add_domain_to_list(&dma_dom->domain);
  1453. if (alloc_new_range(dma_dom, true, GFP_KERNEL))
  1454. goto free_dma_dom;
  1455. /*
  1456. * mark the first page as allocated so we never return 0 as
  1457. * a valid dma-address. So we can use 0 as error value
  1458. */
  1459. dma_dom->aperture[0]->bitmap[0] = 1;
  1460. dma_dom->next_address = 0;
  1461. return dma_dom;
  1462. free_dma_dom:
  1463. dma_ops_domain_free(dma_dom);
  1464. return NULL;
  1465. }
  1466. /*
  1467. * little helper function to check whether a given protection domain is a
  1468. * dma_ops domain
  1469. */
  1470. static bool dma_ops_domain(struct protection_domain *domain)
  1471. {
  1472. return domain->flags & PD_DMA_OPS_MASK;
  1473. }
  1474. static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
  1475. {
  1476. u64 pte_root = 0;
  1477. u64 flags = 0;
  1478. if (domain->mode != PAGE_MODE_NONE)
  1479. pte_root = virt_to_phys(domain->pt_root);
  1480. pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
  1481. << DEV_ENTRY_MODE_SHIFT;
  1482. pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
  1483. flags = amd_iommu_dev_table[devid].data[1];
  1484. if (ats)
  1485. flags |= DTE_FLAG_IOTLB;
  1486. if (domain->flags & PD_IOMMUV2_MASK) {
  1487. u64 gcr3 = __pa(domain->gcr3_tbl);
  1488. u64 glx = domain->glx;
  1489. u64 tmp;
  1490. pte_root |= DTE_FLAG_GV;
  1491. pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
  1492. /* First mask out possible old values for GCR3 table */
  1493. tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
  1494. flags &= ~tmp;
  1495. tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
  1496. flags &= ~tmp;
  1497. /* Encode GCR3 table into DTE */
  1498. tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
  1499. pte_root |= tmp;
  1500. tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
  1501. flags |= tmp;
  1502. tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
  1503. flags |= tmp;
  1504. }
  1505. flags &= ~(0xffffUL);
  1506. flags |= domain->id;
  1507. amd_iommu_dev_table[devid].data[1] = flags;
  1508. amd_iommu_dev_table[devid].data[0] = pte_root;
  1509. }
  1510. static void clear_dte_entry(u16 devid)
  1511. {
  1512. /* remove entry from the device table seen by the hardware */
  1513. amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
  1514. amd_iommu_dev_table[devid].data[1] = 0;
  1515. amd_iommu_apply_erratum_63(devid);
  1516. }
  1517. static void do_attach(struct iommu_dev_data *dev_data,
  1518. struct protection_domain *domain)
  1519. {
  1520. struct amd_iommu *iommu;
  1521. bool ats;
  1522. iommu = amd_iommu_rlookup_table[dev_data->devid];
  1523. ats = dev_data->ats.enabled;
  1524. /* Update data structures */
  1525. dev_data->domain = domain;
  1526. list_add(&dev_data->list, &domain->dev_list);
  1527. set_dte_entry(dev_data->devid, domain, ats);
  1528. /* Do reference counting */
  1529. domain->dev_iommu[iommu->index] += 1;
  1530. domain->dev_cnt += 1;
  1531. /* Flush the DTE entry */
  1532. device_flush_dte(dev_data);
  1533. }
  1534. static void do_detach(struct iommu_dev_data *dev_data)
  1535. {
  1536. struct amd_iommu *iommu;
  1537. iommu = amd_iommu_rlookup_table[dev_data->devid];
  1538. /* decrease reference counters */
  1539. dev_data->domain->dev_iommu[iommu->index] -= 1;
  1540. dev_data->domain->dev_cnt -= 1;
  1541. /* Update data structures */
  1542. dev_data->domain = NULL;
  1543. list_del(&dev_data->list);
  1544. clear_dte_entry(dev_data->devid);
  1545. /* Flush the DTE entry */
  1546. device_flush_dte(dev_data);
  1547. }
  1548. /*
  1549. * If a device is not yet associated with a domain, this function does
  1550. * assigns it visible for the hardware
  1551. */
  1552. static int __attach_device(struct iommu_dev_data *dev_data,
  1553. struct protection_domain *domain)
  1554. {
  1555. int ret;
  1556. /* lock domain */
  1557. spin_lock(&domain->lock);
  1558. if (dev_data->alias_data != NULL) {
  1559. struct iommu_dev_data *alias_data = dev_data->alias_data;
  1560. /* Some sanity checks */
  1561. ret = -EBUSY;
  1562. if (alias_data->domain != NULL &&
  1563. alias_data->domain != domain)
  1564. goto out_unlock;
  1565. if (dev_data->domain != NULL &&
  1566. dev_data->domain != domain)
  1567. goto out_unlock;
  1568. /* Do real assignment */
  1569. if (alias_data->domain == NULL)
  1570. do_attach(alias_data, domain);
  1571. atomic_inc(&alias_data->bind);
  1572. }
  1573. if (dev_data->domain == NULL)
  1574. do_attach(dev_data, domain);
  1575. atomic_inc(&dev_data->bind);
  1576. ret = 0;
  1577. out_unlock:
  1578. /* ready */
  1579. spin_unlock(&domain->lock);
  1580. return ret;
  1581. }
  1582. static void pdev_iommuv2_disable(struct pci_dev *pdev)
  1583. {
  1584. pci_disable_ats(pdev);
  1585. pci_disable_pri(pdev);
  1586. pci_disable_pasid(pdev);
  1587. }
  1588. /* FIXME: Change generic reset-function to do the same */
  1589. static int pri_reset_while_enabled(struct pci_dev *pdev)
  1590. {
  1591. u16 control;
  1592. int pos;
  1593. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
  1594. if (!pos)
  1595. return -EINVAL;
  1596. pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
  1597. control |= PCI_PRI_CTRL_RESET;
  1598. pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
  1599. return 0;
  1600. }
  1601. static int pdev_iommuv2_enable(struct pci_dev *pdev)
  1602. {
  1603. bool reset_enable;
  1604. int reqs, ret;
  1605. /* FIXME: Hardcode number of outstanding requests for now */
  1606. reqs = 32;
  1607. if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
  1608. reqs = 1;
  1609. reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
  1610. /* Only allow access to user-accessible pages */
  1611. ret = pci_enable_pasid(pdev, 0);
  1612. if (ret)
  1613. goto out_err;
  1614. /* First reset the PRI state of the device */
  1615. ret = pci_reset_pri(pdev);
  1616. if (ret)
  1617. goto out_err;
  1618. /* Enable PRI */
  1619. ret = pci_enable_pri(pdev, reqs);
  1620. if (ret)
  1621. goto out_err;
  1622. if (reset_enable) {
  1623. ret = pri_reset_while_enabled(pdev);
  1624. if (ret)
  1625. goto out_err;
  1626. }
  1627. ret = pci_enable_ats(pdev, PAGE_SHIFT);
  1628. if (ret)
  1629. goto out_err;
  1630. return 0;
  1631. out_err:
  1632. pci_disable_pri(pdev);
  1633. pci_disable_pasid(pdev);
  1634. return ret;
  1635. }
  1636. /* FIXME: Move this to PCI code */
  1637. #define PCI_PRI_TLP_OFF (1 << 15)
  1638. bool pci_pri_tlp_required(struct pci_dev *pdev)
  1639. {
  1640. u16 status;
  1641. int pos;
  1642. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
  1643. if (!pos)
  1644. return false;
  1645. pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
  1646. return (status & PCI_PRI_TLP_OFF) ? true : false;
  1647. }
  1648. /*
  1649. * If a device is not yet associated with a domain, this function does
  1650. * assigns it visible for the hardware
  1651. */
  1652. static int attach_device(struct device *dev,
  1653. struct protection_domain *domain)
  1654. {
  1655. struct pci_dev *pdev = to_pci_dev(dev);
  1656. struct iommu_dev_data *dev_data;
  1657. unsigned long flags;
  1658. int ret;
  1659. dev_data = get_dev_data(dev);
  1660. if (domain->flags & PD_IOMMUV2_MASK) {
  1661. if (!dev_data->iommu_v2 || !dev_data->passthrough)
  1662. return -EINVAL;
  1663. if (pdev_iommuv2_enable(pdev) != 0)
  1664. return -EINVAL;
  1665. dev_data->ats.enabled = true;
  1666. dev_data->ats.qdep = pci_ats_queue_depth(pdev);
  1667. dev_data->pri_tlp = pci_pri_tlp_required(pdev);
  1668. } else if (amd_iommu_iotlb_sup &&
  1669. pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
  1670. dev_data->ats.enabled = true;
  1671. dev_data->ats.qdep = pci_ats_queue_depth(pdev);
  1672. }
  1673. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1674. ret = __attach_device(dev_data, domain);
  1675. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1676. /*
  1677. * We might boot into a crash-kernel here. The crashed kernel
  1678. * left the caches in the IOMMU dirty. So we have to flush
  1679. * here to evict all dirty stuff.
  1680. */
  1681. domain_flush_tlb_pde(domain);
  1682. return ret;
  1683. }
  1684. /*
  1685. * Removes a device from a protection domain (unlocked)
  1686. */
  1687. static void __detach_device(struct iommu_dev_data *dev_data)
  1688. {
  1689. struct protection_domain *domain;
  1690. unsigned long flags;
  1691. BUG_ON(!dev_data->domain);
  1692. domain = dev_data->domain;
  1693. spin_lock_irqsave(&domain->lock, flags);
  1694. if (dev_data->alias_data != NULL) {
  1695. struct iommu_dev_data *alias_data = dev_data->alias_data;
  1696. if (atomic_dec_and_test(&alias_data->bind))
  1697. do_detach(alias_data);
  1698. }
  1699. if (atomic_dec_and_test(&dev_data->bind))
  1700. do_detach(dev_data);
  1701. spin_unlock_irqrestore(&domain->lock, flags);
  1702. /*
  1703. * If we run in passthrough mode the device must be assigned to the
  1704. * passthrough domain if it is detached from any other domain.
  1705. * Make sure we can deassign from the pt_domain itself.
  1706. */
  1707. if (dev_data->passthrough &&
  1708. (dev_data->domain == NULL && domain != pt_domain))
  1709. __attach_device(dev_data, pt_domain);
  1710. }
  1711. /*
  1712. * Removes a device from a protection domain (with devtable_lock held)
  1713. */
  1714. static void detach_device(struct device *dev)
  1715. {
  1716. struct protection_domain *domain;
  1717. struct iommu_dev_data *dev_data;
  1718. unsigned long flags;
  1719. dev_data = get_dev_data(dev);
  1720. domain = dev_data->domain;
  1721. /* lock device table */
  1722. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1723. __detach_device(dev_data);
  1724. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1725. if (domain->flags & PD_IOMMUV2_MASK)
  1726. pdev_iommuv2_disable(to_pci_dev(dev));
  1727. else if (dev_data->ats.enabled)
  1728. pci_disable_ats(to_pci_dev(dev));
  1729. dev_data->ats.enabled = false;
  1730. }
  1731. /*
  1732. * Find out the protection domain structure for a given PCI device. This
  1733. * will give us the pointer to the page table root for example.
  1734. */
  1735. static struct protection_domain *domain_for_device(struct device *dev)
  1736. {
  1737. struct iommu_dev_data *dev_data;
  1738. struct protection_domain *dom = NULL;
  1739. unsigned long flags;
  1740. dev_data = get_dev_data(dev);
  1741. if (dev_data->domain)
  1742. return dev_data->domain;
  1743. if (dev_data->alias_data != NULL) {
  1744. struct iommu_dev_data *alias_data = dev_data->alias_data;
  1745. read_lock_irqsave(&amd_iommu_devtable_lock, flags);
  1746. if (alias_data->domain != NULL) {
  1747. __attach_device(dev_data, alias_data->domain);
  1748. dom = alias_data->domain;
  1749. }
  1750. read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  1751. }
  1752. return dom;
  1753. }
  1754. static int device_change_notifier(struct notifier_block *nb,
  1755. unsigned long action, void *data)
  1756. {
  1757. struct dma_ops_domain *dma_domain;
  1758. struct protection_domain *domain;
  1759. struct iommu_dev_data *dev_data;
  1760. struct device *dev = data;
  1761. struct amd_iommu *iommu;
  1762. unsigned long flags;
  1763. u16 devid;
  1764. if (!check_device(dev))
  1765. return 0;
  1766. devid = get_device_id(dev);
  1767. iommu = amd_iommu_rlookup_table[devid];
  1768. dev_data = get_dev_data(dev);
  1769. switch (action) {
  1770. case BUS_NOTIFY_UNBOUND_DRIVER:
  1771. domain = domain_for_device(dev);
  1772. if (!domain)
  1773. goto out;
  1774. if (dev_data->passthrough)
  1775. break;
  1776. detach_device(dev);
  1777. break;
  1778. case BUS_NOTIFY_ADD_DEVICE:
  1779. iommu_init_device(dev);
  1780. domain = domain_for_device(dev);
  1781. /* allocate a protection domain if a device is added */
  1782. dma_domain = find_protection_domain(devid);
  1783. if (dma_domain)
  1784. goto out;
  1785. dma_domain = dma_ops_domain_alloc();
  1786. if (!dma_domain)
  1787. goto out;
  1788. dma_domain->target_dev = devid;
  1789. spin_lock_irqsave(&iommu_pd_list_lock, flags);
  1790. list_add_tail(&dma_domain->list, &iommu_pd_list);
  1791. spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  1792. dev_data = get_dev_data(dev);
  1793. if (!dev_data->passthrough)
  1794. dev->archdata.dma_ops = &amd_iommu_dma_ops;
  1795. else
  1796. dev->archdata.dma_ops = &nommu_dma_ops;
  1797. break;
  1798. case BUS_NOTIFY_DEL_DEVICE:
  1799. iommu_uninit_device(dev);
  1800. default:
  1801. goto out;
  1802. }
  1803. iommu_completion_wait(iommu);
  1804. out:
  1805. return 0;
  1806. }
  1807. static struct notifier_block device_nb = {
  1808. .notifier_call = device_change_notifier,
  1809. };
  1810. void amd_iommu_init_notifier(void)
  1811. {
  1812. bus_register_notifier(&pci_bus_type, &device_nb);
  1813. }
  1814. /*****************************************************************************
  1815. *
  1816. * The next functions belong to the dma_ops mapping/unmapping code.
  1817. *
  1818. *****************************************************************************/
  1819. /*
  1820. * In the dma_ops path we only have the struct device. This function
  1821. * finds the corresponding IOMMU, the protection domain and the
  1822. * requestor id for a given device.
  1823. * If the device is not yet associated with a domain this is also done
  1824. * in this function.
  1825. */
  1826. static struct protection_domain *get_domain(struct device *dev)
  1827. {
  1828. struct protection_domain *domain;
  1829. struct dma_ops_domain *dma_dom;
  1830. u16 devid = get_device_id(dev);
  1831. if (!check_device(dev))
  1832. return ERR_PTR(-EINVAL);
  1833. domain = domain_for_device(dev);
  1834. if (domain != NULL && !dma_ops_domain(domain))
  1835. return ERR_PTR(-EBUSY);
  1836. if (domain != NULL)
  1837. return domain;
  1838. /* Device not bount yet - bind it */
  1839. dma_dom = find_protection_domain(devid);
  1840. if (!dma_dom)
  1841. dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
  1842. attach_device(dev, &dma_dom->domain);
  1843. DUMP_printk("Using protection domain %d for device %s\n",
  1844. dma_dom->domain.id, dev_name(dev));
  1845. return &dma_dom->domain;
  1846. }
  1847. static void update_device_table(struct protection_domain *domain)
  1848. {
  1849. struct iommu_dev_data *dev_data;
  1850. list_for_each_entry(dev_data, &domain->dev_list, list)
  1851. set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
  1852. }
  1853. static void update_domain(struct protection_domain *domain)
  1854. {
  1855. if (!domain->updated)
  1856. return;
  1857. update_device_table(domain);
  1858. domain_flush_devices(domain);
  1859. domain_flush_tlb_pde(domain);
  1860. domain->updated = false;
  1861. }
  1862. /*
  1863. * This function fetches the PTE for a given address in the aperture
  1864. */
  1865. static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
  1866. unsigned long address)
  1867. {
  1868. struct aperture_range *aperture;
  1869. u64 *pte, *pte_page;
  1870. aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
  1871. if (!aperture)
  1872. return NULL;
  1873. pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
  1874. if (!pte) {
  1875. pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
  1876. GFP_ATOMIC);
  1877. aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
  1878. } else
  1879. pte += PM_LEVEL_INDEX(0, address);
  1880. update_domain(&dom->domain);
  1881. return pte;
  1882. }
  1883. /*
  1884. * This is the generic map function. It maps one 4kb page at paddr to
  1885. * the given address in the DMA address space for the domain.
  1886. */
  1887. static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
  1888. unsigned long address,
  1889. phys_addr_t paddr,
  1890. int direction)
  1891. {
  1892. u64 *pte, __pte;
  1893. WARN_ON(address > dom->aperture_size);
  1894. paddr &= PAGE_MASK;
  1895. pte = dma_ops_get_pte(dom, address);
  1896. if (!pte)
  1897. return DMA_ERROR_CODE;
  1898. __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
  1899. if (direction == DMA_TO_DEVICE)
  1900. __pte |= IOMMU_PTE_IR;
  1901. else if (direction == DMA_FROM_DEVICE)
  1902. __pte |= IOMMU_PTE_IW;
  1903. else if (direction == DMA_BIDIRECTIONAL)
  1904. __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
  1905. WARN_ON(*pte);
  1906. *pte = __pte;
  1907. return (dma_addr_t)address;
  1908. }
  1909. /*
  1910. * The generic unmapping function for on page in the DMA address space.
  1911. */
  1912. static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
  1913. unsigned long address)
  1914. {
  1915. struct aperture_range *aperture;
  1916. u64 *pte;
  1917. if (address >= dom->aperture_size)
  1918. return;
  1919. aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
  1920. if (!aperture)
  1921. return;
  1922. pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
  1923. if (!pte)
  1924. return;
  1925. pte += PM_LEVEL_INDEX(0, address);
  1926. WARN_ON(!*pte);
  1927. *pte = 0ULL;
  1928. }
  1929. /*
  1930. * This function contains common code for mapping of a physically
  1931. * contiguous memory region into DMA address space. It is used by all
  1932. * mapping functions provided with this IOMMU driver.
  1933. * Must be called with the domain lock held.
  1934. */
  1935. static dma_addr_t __map_single(struct device *dev,
  1936. struct dma_ops_domain *dma_dom,
  1937. phys_addr_t paddr,
  1938. size_t size,
  1939. int dir,
  1940. bool align,
  1941. u64 dma_mask)
  1942. {
  1943. dma_addr_t offset = paddr & ~PAGE_MASK;
  1944. dma_addr_t address, start, ret;
  1945. unsigned int pages;
  1946. unsigned long align_mask = 0;
  1947. int i;
  1948. pages = iommu_num_pages(paddr, size, PAGE_SIZE);
  1949. paddr &= PAGE_MASK;
  1950. INC_STATS_COUNTER(total_map_requests);
  1951. if (pages > 1)
  1952. INC_STATS_COUNTER(cross_page);
  1953. if (align)
  1954. align_mask = (1UL << get_order(size)) - 1;
  1955. retry:
  1956. address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
  1957. dma_mask);
  1958. if (unlikely(address == DMA_ERROR_CODE)) {
  1959. /*
  1960. * setting next_address here will let the address
  1961. * allocator only scan the new allocated range in the
  1962. * first run. This is a small optimization.
  1963. */
  1964. dma_dom->next_address = dma_dom->aperture_size;
  1965. if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
  1966. goto out;
  1967. /*
  1968. * aperture was successfully enlarged by 128 MB, try
  1969. * allocation again
  1970. */
  1971. goto retry;
  1972. }
  1973. start = address;
  1974. for (i = 0; i < pages; ++i) {
  1975. ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
  1976. if (ret == DMA_ERROR_CODE)
  1977. goto out_unmap;
  1978. paddr += PAGE_SIZE;
  1979. start += PAGE_SIZE;
  1980. }
  1981. address += offset;
  1982. ADD_STATS_COUNTER(alloced_io_mem, size);
  1983. if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
  1984. domain_flush_tlb(&dma_dom->domain);
  1985. dma_dom->need_flush = false;
  1986. } else if (unlikely(amd_iommu_np_cache))
  1987. domain_flush_pages(&dma_dom->domain, address, size);
  1988. out:
  1989. return address;
  1990. out_unmap:
  1991. for (--i; i >= 0; --i) {
  1992. start -= PAGE_SIZE;
  1993. dma_ops_domain_unmap(dma_dom, start);
  1994. }
  1995. dma_ops_free_addresses(dma_dom, address, pages);
  1996. return DMA_ERROR_CODE;
  1997. }
  1998. /*
  1999. * Does the reverse of the __map_single function. Must be called with
  2000. * the domain lock held too
  2001. */
  2002. static void __unmap_single(struct dma_ops_domain *dma_dom,
  2003. dma_addr_t dma_addr,
  2004. size_t size,
  2005. int dir)
  2006. {
  2007. dma_addr_t flush_addr;
  2008. dma_addr_t i, start;
  2009. unsigned int pages;
  2010. if ((dma_addr == DMA_ERROR_CODE) ||
  2011. (dma_addr + size > dma_dom->aperture_size))
  2012. return;
  2013. flush_addr = dma_addr;
  2014. pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
  2015. dma_addr &= PAGE_MASK;
  2016. start = dma_addr;
  2017. for (i = 0; i < pages; ++i) {
  2018. dma_ops_domain_unmap(dma_dom, start);
  2019. start += PAGE_SIZE;
  2020. }
  2021. SUB_STATS_COUNTER(alloced_io_mem, size);
  2022. dma_ops_free_addresses(dma_dom, dma_addr, pages);
  2023. if (amd_iommu_unmap_flush || dma_dom->need_flush) {
  2024. domain_flush_pages(&dma_dom->domain, flush_addr, size);
  2025. dma_dom->need_flush = false;
  2026. }
  2027. }
  2028. /*
  2029. * The exported map_single function for dma_ops.
  2030. */
  2031. static dma_addr_t map_page(struct device *dev, struct page *page,
  2032. unsigned long offset, size_t size,
  2033. enum dma_data_direction dir,
  2034. struct dma_attrs *attrs)
  2035. {
  2036. unsigned long flags;
  2037. struct protection_domain *domain;
  2038. dma_addr_t addr;
  2039. u64 dma_mask;
  2040. phys_addr_t paddr = page_to_phys(page) + offset;
  2041. INC_STATS_COUNTER(cnt_map_single);
  2042. domain = get_domain(dev);
  2043. if (PTR_ERR(domain) == -EINVAL)
  2044. return (dma_addr_t)paddr;
  2045. else if (IS_ERR(domain))
  2046. return DMA_ERROR_CODE;
  2047. dma_mask = *dev->dma_mask;
  2048. spin_lock_irqsave(&domain->lock, flags);
  2049. addr = __map_single(dev, domain->priv, paddr, size, dir, false,
  2050. dma_mask);
  2051. if (addr == DMA_ERROR_CODE)
  2052. goto out;
  2053. domain_flush_complete(domain);
  2054. out:
  2055. spin_unlock_irqrestore(&domain->lock, flags);
  2056. return addr;
  2057. }
  2058. /*
  2059. * The exported unmap_single function for dma_ops.
  2060. */
  2061. static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
  2062. enum dma_data_direction dir, struct dma_attrs *attrs)
  2063. {
  2064. unsigned long flags;
  2065. struct protection_domain *domain;
  2066. INC_STATS_COUNTER(cnt_unmap_single);
  2067. domain = get_domain(dev);
  2068. if (IS_ERR(domain))
  2069. return;
  2070. spin_lock_irqsave(&domain->lock, flags);
  2071. __unmap_single(domain->priv, dma_addr, size, dir);
  2072. domain_flush_complete(domain);
  2073. spin_unlock_irqrestore(&domain->lock, flags);
  2074. }
  2075. /*
  2076. * This is a special map_sg function which is used if we should map a
  2077. * device which is not handled by an AMD IOMMU in the system.
  2078. */
  2079. static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
  2080. int nelems, int dir)
  2081. {
  2082. struct scatterlist *s;
  2083. int i;
  2084. for_each_sg(sglist, s, nelems, i) {
  2085. s->dma_address = (dma_addr_t)sg_phys(s);
  2086. s->dma_length = s->length;
  2087. }
  2088. return nelems;
  2089. }
  2090. /*
  2091. * The exported map_sg function for dma_ops (handles scatter-gather
  2092. * lists).
  2093. */
  2094. static int map_sg(struct device *dev, struct scatterlist *sglist,
  2095. int nelems, enum dma_data_direction dir,
  2096. struct dma_attrs *attrs)
  2097. {
  2098. unsigned long flags;
  2099. struct protection_domain *domain;
  2100. int i;
  2101. struct scatterlist *s;
  2102. phys_addr_t paddr;
  2103. int mapped_elems = 0;
  2104. u64 dma_mask;
  2105. INC_STATS_COUNTER(cnt_map_sg);
  2106. domain = get_domain(dev);
  2107. if (PTR_ERR(domain) == -EINVAL)
  2108. return map_sg_no_iommu(dev, sglist, nelems, dir);
  2109. else if (IS_ERR(domain))
  2110. return 0;
  2111. dma_mask = *dev->dma_mask;
  2112. spin_lock_irqsave(&domain->lock, flags);
  2113. for_each_sg(sglist, s, nelems, i) {
  2114. paddr = sg_phys(s);
  2115. s->dma_address = __map_single(dev, domain->priv,
  2116. paddr, s->length, dir, false,
  2117. dma_mask);
  2118. if (s->dma_address) {
  2119. s->dma_length = s->length;
  2120. mapped_elems++;
  2121. } else
  2122. goto unmap;
  2123. }
  2124. domain_flush_complete(domain);
  2125. out:
  2126. spin_unlock_irqrestore(&domain->lock, flags);
  2127. return mapped_elems;
  2128. unmap:
  2129. for_each_sg(sglist, s, mapped_elems, i) {
  2130. if (s->dma_address)
  2131. __unmap_single(domain->priv, s->dma_address,
  2132. s->dma_length, dir);
  2133. s->dma_address = s->dma_length = 0;
  2134. }
  2135. mapped_elems = 0;
  2136. goto out;
  2137. }
  2138. /*
  2139. * The exported map_sg function for dma_ops (handles scatter-gather
  2140. * lists).
  2141. */
  2142. static void unmap_sg(struct device *dev, struct scatterlist *sglist,
  2143. int nelems, enum dma_data_direction dir,
  2144. struct dma_attrs *attrs)
  2145. {
  2146. unsigned long flags;
  2147. struct protection_domain *domain;
  2148. struct scatterlist *s;
  2149. int i;
  2150. INC_STATS_COUNTER(cnt_unmap_sg);
  2151. domain = get_domain(dev);
  2152. if (IS_ERR(domain))
  2153. return;
  2154. spin_lock_irqsave(&domain->lock, flags);
  2155. for_each_sg(sglist, s, nelems, i) {
  2156. __unmap_single(domain->priv, s->dma_address,
  2157. s->dma_length, dir);
  2158. s->dma_address = s->dma_length = 0;
  2159. }
  2160. domain_flush_complete(domain);
  2161. spin_unlock_irqrestore(&domain->lock, flags);
  2162. }
  2163. /*
  2164. * The exported alloc_coherent function for dma_ops.
  2165. */
  2166. static void *alloc_coherent(struct device *dev, size_t size,
  2167. dma_addr_t *dma_addr, gfp_t flag,
  2168. struct dma_attrs *attrs)
  2169. {
  2170. unsigned long flags;
  2171. void *virt_addr;
  2172. struct protection_domain *domain;
  2173. phys_addr_t paddr;
  2174. u64 dma_mask = dev->coherent_dma_mask;
  2175. INC_STATS_COUNTER(cnt_alloc_coherent);
  2176. domain = get_domain(dev);
  2177. if (PTR_ERR(domain) == -EINVAL) {
  2178. virt_addr = (void *)__get_free_pages(flag, get_order(size));
  2179. *dma_addr = __pa(virt_addr);
  2180. return virt_addr;
  2181. } else if (IS_ERR(domain))
  2182. return NULL;
  2183. dma_mask = dev->coherent_dma_mask;
  2184. flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
  2185. flag |= __GFP_ZERO;
  2186. virt_addr = (void *)__get_free_pages(flag, get_order(size));
  2187. if (!virt_addr)
  2188. return NULL;
  2189. paddr = virt_to_phys(virt_addr);
  2190. if (!dma_mask)
  2191. dma_mask = *dev->dma_mask;
  2192. spin_lock_irqsave(&domain->lock, flags);
  2193. *dma_addr = __map_single(dev, domain->priv, paddr,
  2194. size, DMA_BIDIRECTIONAL, true, dma_mask);
  2195. if (*dma_addr == DMA_ERROR_CODE) {
  2196. spin_unlock_irqrestore(&domain->lock, flags);
  2197. goto out_free;
  2198. }
  2199. domain_flush_complete(domain);
  2200. spin_unlock_irqrestore(&domain->lock, flags);
  2201. return virt_addr;
  2202. out_free:
  2203. free_pages((unsigned long)virt_addr, get_order(size));
  2204. return NULL;
  2205. }
  2206. /*
  2207. * The exported free_coherent function for dma_ops.
  2208. */
  2209. static void free_coherent(struct device *dev, size_t size,
  2210. void *virt_addr, dma_addr_t dma_addr,
  2211. struct dma_attrs *attrs)
  2212. {
  2213. unsigned long flags;
  2214. struct protection_domain *domain;
  2215. INC_STATS_COUNTER(cnt_free_coherent);
  2216. domain = get_domain(dev);
  2217. if (IS_ERR(domain))
  2218. goto free_mem;
  2219. spin_lock_irqsave(&domain->lock, flags);
  2220. __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
  2221. domain_flush_complete(domain);
  2222. spin_unlock_irqrestore(&domain->lock, flags);
  2223. free_mem:
  2224. free_pages((unsigned long)virt_addr, get_order(size));
  2225. }
  2226. /*
  2227. * This function is called by the DMA layer to find out if we can handle a
  2228. * particular device. It is part of the dma_ops.
  2229. */
  2230. static int amd_iommu_dma_supported(struct device *dev, u64 mask)
  2231. {
  2232. return check_device(dev);
  2233. }
  2234. /*
  2235. * The function for pre-allocating protection domains.
  2236. *
  2237. * If the driver core informs the DMA layer if a driver grabs a device
  2238. * we don't need to preallocate the protection domains anymore.
  2239. * For now we have to.
  2240. */
  2241. static void __init prealloc_protection_domains(void)
  2242. {
  2243. struct iommu_dev_data *dev_data;
  2244. struct dma_ops_domain *dma_dom;
  2245. struct pci_dev *dev = NULL;
  2246. u16 devid;
  2247. for_each_pci_dev(dev) {
  2248. /* Do we handle this device? */
  2249. if (!check_device(&dev->dev))
  2250. continue;
  2251. dev_data = get_dev_data(&dev->dev);
  2252. if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
  2253. /* Make sure passthrough domain is allocated */
  2254. alloc_passthrough_domain();
  2255. dev_data->passthrough = true;
  2256. attach_device(&dev->dev, pt_domain);
  2257. pr_info("AMD-Vi: Using passthough domain for device %s\n",
  2258. dev_name(&dev->dev));
  2259. }
  2260. /* Is there already any domain for it? */
  2261. if (domain_for_device(&dev->dev))
  2262. continue;
  2263. devid = get_device_id(&dev->dev);
  2264. dma_dom = dma_ops_domain_alloc();
  2265. if (!dma_dom)
  2266. continue;
  2267. init_unity_mappings_for_device(dma_dom, devid);
  2268. dma_dom->target_dev = devid;
  2269. attach_device(&dev->dev, &dma_dom->domain);
  2270. list_add_tail(&dma_dom->list, &iommu_pd_list);
  2271. }
  2272. }
  2273. static struct dma_map_ops amd_iommu_dma_ops = {
  2274. .alloc = alloc_coherent,
  2275. .free = free_coherent,
  2276. .map_page = map_page,
  2277. .unmap_page = unmap_page,
  2278. .map_sg = map_sg,
  2279. .unmap_sg = unmap_sg,
  2280. .dma_supported = amd_iommu_dma_supported,
  2281. };
  2282. static unsigned device_dma_ops_init(void)
  2283. {
  2284. struct iommu_dev_data *dev_data;
  2285. struct pci_dev *pdev = NULL;
  2286. unsigned unhandled = 0;
  2287. for_each_pci_dev(pdev) {
  2288. if (!check_device(&pdev->dev)) {
  2289. iommu_ignore_device(&pdev->dev);
  2290. unhandled += 1;
  2291. continue;
  2292. }
  2293. dev_data = get_dev_data(&pdev->dev);
  2294. if (!dev_data->passthrough)
  2295. pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
  2296. else
  2297. pdev->dev.archdata.dma_ops = &nommu_dma_ops;
  2298. }
  2299. return unhandled;
  2300. }
  2301. /*
  2302. * The function which clues the AMD IOMMU driver into dma_ops.
  2303. */
  2304. void __init amd_iommu_init_api(void)
  2305. {
  2306. bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
  2307. }
  2308. int __init amd_iommu_init_dma_ops(void)
  2309. {
  2310. struct amd_iommu *iommu;
  2311. int ret, unhandled;
  2312. /*
  2313. * first allocate a default protection domain for every IOMMU we
  2314. * found in the system. Devices not assigned to any other
  2315. * protection domain will be assigned to the default one.
  2316. */
  2317. for_each_iommu(iommu) {
  2318. iommu->default_dom = dma_ops_domain_alloc();
  2319. if (iommu->default_dom == NULL)
  2320. return -ENOMEM;
  2321. iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
  2322. ret = iommu_init_unity_mappings(iommu);
  2323. if (ret)
  2324. goto free_domains;
  2325. }
  2326. /*
  2327. * Pre-allocate the protection domains for each device.
  2328. */
  2329. prealloc_protection_domains();
  2330. iommu_detected = 1;
  2331. swiotlb = 0;
  2332. /* Make the driver finally visible to the drivers */
  2333. unhandled = device_dma_ops_init();
  2334. if (unhandled && max_pfn > MAX_DMA32_PFN) {
  2335. /* There are unhandled devices - initialize swiotlb for them */
  2336. swiotlb = 1;
  2337. }
  2338. amd_iommu_stats_init();
  2339. return 0;
  2340. free_domains:
  2341. for_each_iommu(iommu) {
  2342. if (iommu->default_dom)
  2343. dma_ops_domain_free(iommu->default_dom);
  2344. }
  2345. return ret;
  2346. }
  2347. /*****************************************************************************
  2348. *
  2349. * The following functions belong to the exported interface of AMD IOMMU
  2350. *
  2351. * This interface allows access to lower level functions of the IOMMU
  2352. * like protection domain handling and assignement of devices to domains
  2353. * which is not possible with the dma_ops interface.
  2354. *
  2355. *****************************************************************************/
  2356. static void cleanup_domain(struct protection_domain *domain)
  2357. {
  2358. struct iommu_dev_data *dev_data, *next;
  2359. unsigned long flags;
  2360. write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  2361. list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
  2362. __detach_device(dev_data);
  2363. atomic_set(&dev_data->bind, 0);
  2364. }
  2365. write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  2366. }
  2367. static void protection_domain_free(struct protection_domain *domain)
  2368. {
  2369. if (!domain)
  2370. return;
  2371. del_domain_from_list(domain);
  2372. if (domain->id)
  2373. domain_id_free(domain->id);
  2374. kfree(domain);
  2375. }
  2376. static struct protection_domain *protection_domain_alloc(void)
  2377. {
  2378. struct protection_domain *domain;
  2379. domain = kzalloc(sizeof(*domain), GFP_KERNEL);
  2380. if (!domain)
  2381. return NULL;
  2382. spin_lock_init(&domain->lock);
  2383. mutex_init(&domain->api_lock);
  2384. domain->id = domain_id_alloc();
  2385. if (!domain->id)
  2386. goto out_err;
  2387. INIT_LIST_HEAD(&domain->dev_list);
  2388. add_domain_to_list(domain);
  2389. return domain;
  2390. out_err:
  2391. kfree(domain);
  2392. return NULL;
  2393. }
  2394. static int __init alloc_passthrough_domain(void)
  2395. {
  2396. if (pt_domain != NULL)
  2397. return 0;
  2398. /* allocate passthrough domain */
  2399. pt_domain = protection_domain_alloc();
  2400. if (!pt_domain)
  2401. return -ENOMEM;
  2402. pt_domain->mode = PAGE_MODE_NONE;
  2403. return 0;
  2404. }
  2405. static int amd_iommu_domain_init(struct iommu_domain *dom)
  2406. {
  2407. struct protection_domain *domain;
  2408. domain = protection_domain_alloc();
  2409. if (!domain)
  2410. goto out_free;
  2411. domain->mode = PAGE_MODE_3_LEVEL;
  2412. domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
  2413. if (!domain->pt_root)
  2414. goto out_free;
  2415. domain->iommu_domain = dom;
  2416. dom->priv = domain;
  2417. return 0;
  2418. out_free:
  2419. protection_domain_free(domain);
  2420. return -ENOMEM;
  2421. }
  2422. static void amd_iommu_domain_destroy(struct iommu_domain *dom)
  2423. {
  2424. struct protection_domain *domain = dom->priv;
  2425. if (!domain)
  2426. return;
  2427. if (domain->dev_cnt > 0)
  2428. cleanup_domain(domain);
  2429. BUG_ON(domain->dev_cnt != 0);
  2430. if (domain->mode != PAGE_MODE_NONE)
  2431. free_pagetable(domain);
  2432. if (domain->flags & PD_IOMMUV2_MASK)
  2433. free_gcr3_table(domain);
  2434. protection_domain_free(domain);
  2435. dom->priv = NULL;
  2436. }
  2437. static void amd_iommu_detach_device(struct iommu_domain *dom,
  2438. struct device *dev)
  2439. {
  2440. struct iommu_dev_data *dev_data = dev->archdata.iommu;
  2441. struct amd_iommu *iommu;
  2442. u16 devid;
  2443. if (!check_device(dev))
  2444. return;
  2445. devid = get_device_id(dev);
  2446. if (dev_data->domain != NULL)
  2447. detach_device(dev);
  2448. iommu = amd_iommu_rlookup_table[devid];
  2449. if (!iommu)
  2450. return;
  2451. iommu_completion_wait(iommu);
  2452. }
  2453. static int amd_iommu_attach_device(struct iommu_domain *dom,
  2454. struct device *dev)
  2455. {
  2456. struct protection_domain *domain = dom->priv;
  2457. struct iommu_dev_data *dev_data;
  2458. struct amd_iommu *iommu;
  2459. int ret;
  2460. if (!check_device(dev))
  2461. return -EINVAL;
  2462. dev_data = dev->archdata.iommu;
  2463. iommu = amd_iommu_rlookup_table[dev_data->devid];
  2464. if (!iommu)
  2465. return -EINVAL;
  2466. if (dev_data->domain)
  2467. detach_device(dev);
  2468. ret = attach_device(dev, domain);
  2469. iommu_completion_wait(iommu);
  2470. return ret;
  2471. }
  2472. static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
  2473. phys_addr_t paddr, size_t page_size, int iommu_prot)
  2474. {
  2475. struct protection_domain *domain = dom->priv;
  2476. int prot = 0;
  2477. int ret;
  2478. if (domain->mode == PAGE_MODE_NONE)
  2479. return -EINVAL;
  2480. if (iommu_prot & IOMMU_READ)
  2481. prot |= IOMMU_PROT_IR;
  2482. if (iommu_prot & IOMMU_WRITE)
  2483. prot |= IOMMU_PROT_IW;
  2484. mutex_lock(&domain->api_lock);
  2485. ret = iommu_map_page(domain, iova, paddr, prot, page_size);
  2486. mutex_unlock(&domain->api_lock);
  2487. return ret;
  2488. }
  2489. static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
  2490. size_t page_size)
  2491. {
  2492. struct protection_domain *domain = dom->priv;
  2493. size_t unmap_size;
  2494. if (domain->mode == PAGE_MODE_NONE)
  2495. return -EINVAL;
  2496. mutex_lock(&domain->api_lock);
  2497. unmap_size = iommu_unmap_page(domain, iova, page_size);
  2498. mutex_unlock(&domain->api_lock);
  2499. domain_flush_tlb_pde(domain);
  2500. return unmap_size;
  2501. }
  2502. static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
  2503. unsigned long iova)
  2504. {
  2505. struct protection_domain *domain = dom->priv;
  2506. unsigned long offset_mask;
  2507. phys_addr_t paddr;
  2508. u64 *pte, __pte;
  2509. if (domain->mode == PAGE_MODE_NONE)
  2510. return iova;
  2511. pte = fetch_pte(domain, iova);
  2512. if (!pte || !IOMMU_PTE_PRESENT(*pte))
  2513. return 0;
  2514. if (PM_PTE_LEVEL(*pte) == 0)
  2515. offset_mask = PAGE_SIZE - 1;
  2516. else
  2517. offset_mask = PTE_PAGE_SIZE(*pte) - 1;
  2518. __pte = *pte & PM_ADDR_MASK;
  2519. paddr = (__pte & ~offset_mask) | (iova & offset_mask);
  2520. return paddr;
  2521. }
  2522. static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
  2523. unsigned long cap)
  2524. {
  2525. switch (cap) {
  2526. case IOMMU_CAP_CACHE_COHERENCY:
  2527. return 1;
  2528. }
  2529. return 0;
  2530. }
  2531. static int amd_iommu_device_group(struct device *dev, unsigned int *groupid)
  2532. {
  2533. struct iommu_dev_data *dev_data = dev->archdata.iommu;
  2534. struct pci_dev *pdev = to_pci_dev(dev);
  2535. u16 devid;
  2536. if (!dev_data)
  2537. return -ENODEV;
  2538. if (pdev->is_virtfn || !iommu_group_mf)
  2539. devid = dev_data->devid;
  2540. else
  2541. devid = calc_devid(pdev->bus->number,
  2542. PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
  2543. *groupid = amd_iommu_alias_table[devid];
  2544. return 0;
  2545. }
  2546. static struct iommu_ops amd_iommu_ops = {
  2547. .domain_init = amd_iommu_domain_init,
  2548. .domain_destroy = amd_iommu_domain_destroy,
  2549. .attach_dev = amd_iommu_attach_device,
  2550. .detach_dev = amd_iommu_detach_device,
  2551. .map = amd_iommu_map,
  2552. .unmap = amd_iommu_unmap,
  2553. .iova_to_phys = amd_iommu_iova_to_phys,
  2554. .domain_has_cap = amd_iommu_domain_has_cap,
  2555. .device_group = amd_iommu_device_group,
  2556. .pgsize_bitmap = AMD_IOMMU_PGSIZES,
  2557. };
  2558. /*****************************************************************************
  2559. *
  2560. * The next functions do a basic initialization of IOMMU for pass through
  2561. * mode
  2562. *
  2563. * In passthrough mode the IOMMU is initialized and enabled but not used for
  2564. * DMA-API translation.
  2565. *
  2566. *****************************************************************************/
  2567. int __init amd_iommu_init_passthrough(void)
  2568. {
  2569. struct iommu_dev_data *dev_data;
  2570. struct pci_dev *dev = NULL;
  2571. struct amd_iommu *iommu;
  2572. u16 devid;
  2573. int ret;
  2574. ret = alloc_passthrough_domain();
  2575. if (ret)
  2576. return ret;
  2577. for_each_pci_dev(dev) {
  2578. if (!check_device(&dev->dev))
  2579. continue;
  2580. dev_data = get_dev_data(&dev->dev);
  2581. dev_data->passthrough = true;
  2582. devid = get_device_id(&dev->dev);
  2583. iommu = amd_iommu_rlookup_table[devid];
  2584. if (!iommu)
  2585. continue;
  2586. attach_device(&dev->dev, pt_domain);
  2587. }
  2588. amd_iommu_stats_init();
  2589. pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
  2590. return 0;
  2591. }
  2592. /* IOMMUv2 specific functions */
  2593. int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
  2594. {
  2595. return atomic_notifier_chain_register(&ppr_notifier, nb);
  2596. }
  2597. EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
  2598. int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
  2599. {
  2600. return atomic_notifier_chain_unregister(&ppr_notifier, nb);
  2601. }
  2602. EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
  2603. void amd_iommu_domain_direct_map(struct iommu_domain *dom)
  2604. {
  2605. struct protection_domain *domain = dom->priv;
  2606. unsigned long flags;
  2607. spin_lock_irqsave(&domain->lock, flags);
  2608. /* Update data structure */
  2609. domain->mode = PAGE_MODE_NONE;
  2610. domain->updated = true;
  2611. /* Make changes visible to IOMMUs */
  2612. update_domain(domain);
  2613. /* Page-table is not visible to IOMMU anymore, so free it */
  2614. free_pagetable(domain);
  2615. spin_unlock_irqrestore(&domain->lock, flags);
  2616. }
  2617. EXPORT_SYMBOL(amd_iommu_domain_direct_map);
  2618. int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
  2619. {
  2620. struct protection_domain *domain = dom->priv;
  2621. unsigned long flags;
  2622. int levels, ret;
  2623. if (pasids <= 0 || pasids > (PASID_MASK + 1))
  2624. return -EINVAL;
  2625. /* Number of GCR3 table levels required */
  2626. for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
  2627. levels += 1;
  2628. if (levels > amd_iommu_max_glx_val)
  2629. return -EINVAL;
  2630. spin_lock_irqsave(&domain->lock, flags);
  2631. /*
  2632. * Save us all sanity checks whether devices already in the
  2633. * domain support IOMMUv2. Just force that the domain has no
  2634. * devices attached when it is switched into IOMMUv2 mode.
  2635. */
  2636. ret = -EBUSY;
  2637. if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
  2638. goto out;
  2639. ret = -ENOMEM;
  2640. domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
  2641. if (domain->gcr3_tbl == NULL)
  2642. goto out;
  2643. domain->glx = levels;
  2644. domain->flags |= PD_IOMMUV2_MASK;
  2645. domain->updated = true;
  2646. update_domain(domain);
  2647. ret = 0;
  2648. out:
  2649. spin_unlock_irqrestore(&domain->lock, flags);
  2650. return ret;
  2651. }
  2652. EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
  2653. static int __flush_pasid(struct protection_domain *domain, int pasid,
  2654. u64 address, bool size)
  2655. {
  2656. struct iommu_dev_data *dev_data;
  2657. struct iommu_cmd cmd;
  2658. int i, ret;
  2659. if (!(domain->flags & PD_IOMMUV2_MASK))
  2660. return -EINVAL;
  2661. build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
  2662. /*
  2663. * IOMMU TLB needs to be flushed before Device TLB to
  2664. * prevent device TLB refill from IOMMU TLB
  2665. */
  2666. for (i = 0; i < amd_iommus_present; ++i) {
  2667. if (domain->dev_iommu[i] == 0)
  2668. continue;
  2669. ret = iommu_queue_command(amd_iommus[i], &cmd);
  2670. if (ret != 0)
  2671. goto out;
  2672. }
  2673. /* Wait until IOMMU TLB flushes are complete */
  2674. domain_flush_complete(domain);
  2675. /* Now flush device TLBs */
  2676. list_for_each_entry(dev_data, &domain->dev_list, list) {
  2677. struct amd_iommu *iommu;
  2678. int qdep;
  2679. BUG_ON(!dev_data->ats.enabled);
  2680. qdep = dev_data->ats.qdep;
  2681. iommu = amd_iommu_rlookup_table[dev_data->devid];
  2682. build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
  2683. qdep, address, size);
  2684. ret = iommu_queue_command(iommu, &cmd);
  2685. if (ret != 0)
  2686. goto out;
  2687. }
  2688. /* Wait until all device TLBs are flushed */
  2689. domain_flush_complete(domain);
  2690. ret = 0;
  2691. out:
  2692. return ret;
  2693. }
  2694. static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
  2695. u64 address)
  2696. {
  2697. INC_STATS_COUNTER(invalidate_iotlb);
  2698. return __flush_pasid(domain, pasid, address, false);
  2699. }
  2700. int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
  2701. u64 address)
  2702. {
  2703. struct protection_domain *domain = dom->priv;
  2704. unsigned long flags;
  2705. int ret;
  2706. spin_lock_irqsave(&domain->lock, flags);
  2707. ret = __amd_iommu_flush_page(domain, pasid, address);
  2708. spin_unlock_irqrestore(&domain->lock, flags);
  2709. return ret;
  2710. }
  2711. EXPORT_SYMBOL(amd_iommu_flush_page);
  2712. static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
  2713. {
  2714. INC_STATS_COUNTER(invalidate_iotlb_all);
  2715. return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
  2716. true);
  2717. }
  2718. int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
  2719. {
  2720. struct protection_domain *domain = dom->priv;
  2721. unsigned long flags;
  2722. int ret;
  2723. spin_lock_irqsave(&domain->lock, flags);
  2724. ret = __amd_iommu_flush_tlb(domain, pasid);
  2725. spin_unlock_irqrestore(&domain->lock, flags);
  2726. return ret;
  2727. }
  2728. EXPORT_SYMBOL(amd_iommu_flush_tlb);
  2729. static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
  2730. {
  2731. int index;
  2732. u64 *pte;
  2733. while (true) {
  2734. index = (pasid >> (9 * level)) & 0x1ff;
  2735. pte = &root[index];
  2736. if (level == 0)
  2737. break;
  2738. if (!(*pte & GCR3_VALID)) {
  2739. if (!alloc)
  2740. return NULL;
  2741. root = (void *)get_zeroed_page(GFP_ATOMIC);
  2742. if (root == NULL)
  2743. return NULL;
  2744. *pte = __pa(root) | GCR3_VALID;
  2745. }
  2746. root = __va(*pte & PAGE_MASK);
  2747. level -= 1;
  2748. }
  2749. return pte;
  2750. }
  2751. static int __set_gcr3(struct protection_domain *domain, int pasid,
  2752. unsigned long cr3)
  2753. {
  2754. u64 *pte;
  2755. if (domain->mode != PAGE_MODE_NONE)
  2756. return -EINVAL;
  2757. pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
  2758. if (pte == NULL)
  2759. return -ENOMEM;
  2760. *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
  2761. return __amd_iommu_flush_tlb(domain, pasid);
  2762. }
  2763. static int __clear_gcr3(struct protection_domain *domain, int pasid)
  2764. {
  2765. u64 *pte;
  2766. if (domain->mode != PAGE_MODE_NONE)
  2767. return -EINVAL;
  2768. pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
  2769. if (pte == NULL)
  2770. return 0;
  2771. *pte = 0;
  2772. return __amd_iommu_flush_tlb(domain, pasid);
  2773. }
  2774. int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
  2775. unsigned long cr3)
  2776. {
  2777. struct protection_domain *domain = dom->priv;
  2778. unsigned long flags;
  2779. int ret;
  2780. spin_lock_irqsave(&domain->lock, flags);
  2781. ret = __set_gcr3(domain, pasid, cr3);
  2782. spin_unlock_irqrestore(&domain->lock, flags);
  2783. return ret;
  2784. }
  2785. EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
  2786. int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
  2787. {
  2788. struct protection_domain *domain = dom->priv;
  2789. unsigned long flags;
  2790. int ret;
  2791. spin_lock_irqsave(&domain->lock, flags);
  2792. ret = __clear_gcr3(domain, pasid);
  2793. spin_unlock_irqrestore(&domain->lock, flags);
  2794. return ret;
  2795. }
  2796. EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
  2797. int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
  2798. int status, int tag)
  2799. {
  2800. struct iommu_dev_data *dev_data;
  2801. struct amd_iommu *iommu;
  2802. struct iommu_cmd cmd;
  2803. INC_STATS_COUNTER(complete_ppr);
  2804. dev_data = get_dev_data(&pdev->dev);
  2805. iommu = amd_iommu_rlookup_table[dev_data->devid];
  2806. build_complete_ppr(&cmd, dev_data->devid, pasid, status,
  2807. tag, dev_data->pri_tlp);
  2808. return iommu_queue_command(iommu, &cmd);
  2809. }
  2810. EXPORT_SYMBOL(amd_iommu_complete_ppr);
  2811. struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
  2812. {
  2813. struct protection_domain *domain;
  2814. domain = get_domain(&pdev->dev);
  2815. if (IS_ERR(domain))
  2816. return NULL;
  2817. /* Only return IOMMUv2 domains */
  2818. if (!(domain->flags & PD_IOMMUV2_MASK))
  2819. return NULL;
  2820. return domain->iommu_domain;
  2821. }
  2822. EXPORT_SYMBOL(amd_iommu_get_v2_domain);
  2823. void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
  2824. {
  2825. struct iommu_dev_data *dev_data;
  2826. if (!amd_iommu_v2_supported())
  2827. return;
  2828. dev_data = get_dev_data(&pdev->dev);
  2829. dev_data->errata |= (1 << erratum);
  2830. }
  2831. EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
  2832. int amd_iommu_device_info(struct pci_dev *pdev,
  2833. struct amd_iommu_device_info *info)
  2834. {
  2835. int max_pasids;
  2836. int pos;
  2837. if (pdev == NULL || info == NULL)
  2838. return -EINVAL;
  2839. if (!amd_iommu_v2_supported())
  2840. return -EINVAL;
  2841. memset(info, 0, sizeof(*info));
  2842. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
  2843. if (pos)
  2844. info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
  2845. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
  2846. if (pos)
  2847. info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
  2848. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
  2849. if (pos) {
  2850. int features;
  2851. max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
  2852. max_pasids = min(max_pasids, (1 << 20));
  2853. info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
  2854. info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
  2855. features = pci_pasid_features(pdev);
  2856. if (features & PCI_PASID_CAP_EXEC)
  2857. info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
  2858. if (features & PCI_PASID_CAP_PRIV)
  2859. info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
  2860. }
  2861. return 0;
  2862. }
  2863. EXPORT_SYMBOL(amd_iommu_device_info);