omap-iommu.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277
  1. /*
  2. * omap iommu: tlb and pagetable primitives
  3. *
  4. * Copyright (C) 2008-2010 Nokia Corporation
  5. *
  6. * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
  7. * Paul Mundt and Toshihiro Kobayashi
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License version 2 as
  11. * published by the Free Software Foundation.
  12. */
  13. #include <linux/err.h>
  14. #include <linux/module.h>
  15. #include <linux/slab.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/ioport.h>
  18. #include <linux/clk.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/iommu.h>
  21. #include <linux/mutex.h>
  22. #include <linux/spinlock.h>
  23. #include <asm/cacheflush.h>
  24. #include <plat/iommu.h>
  25. #include <plat/iopgtable.h>
  26. #define for_each_iotlb_cr(obj, n, __i, cr) \
  27. for (__i = 0; \
  28. (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
  29. __i++)
  30. /**
  31. * struct omap_iommu_domain - omap iommu domain
  32. * @pgtable: the page table
  33. * @iommu_dev: an omap iommu device attached to this domain. only a single
  34. * iommu device can be attached for now.
  35. * @lock: domain lock, should be taken when attaching/detaching
  36. */
  37. struct omap_iommu_domain {
  38. u32 *pgtable;
  39. struct omap_iommu *iommu_dev;
  40. spinlock_t lock;
  41. };
  42. /* accommodate the difference between omap1 and omap2/3 */
  43. static const struct iommu_functions *arch_iommu;
  44. static struct platform_driver omap_iommu_driver;
  45. static struct kmem_cache *iopte_cachep;
  46. /**
  47. * omap_install_iommu_arch - Install archtecure specific iommu functions
  48. * @ops: a pointer to architecture specific iommu functions
  49. *
  50. * There are several kind of iommu algorithm(tlb, pagetable) among
  51. * omap series. This interface installs such an iommu algorighm.
  52. **/
  53. int omap_install_iommu_arch(const struct iommu_functions *ops)
  54. {
  55. if (arch_iommu)
  56. return -EBUSY;
  57. arch_iommu = ops;
  58. return 0;
  59. }
  60. EXPORT_SYMBOL_GPL(omap_install_iommu_arch);
  61. /**
  62. * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions
  63. * @ops: a pointer to architecture specific iommu functions
  64. *
  65. * This interface uninstalls the iommu algorighm installed previously.
  66. **/
  67. void omap_uninstall_iommu_arch(const struct iommu_functions *ops)
  68. {
  69. if (arch_iommu != ops)
  70. pr_err("%s: not your arch\n", __func__);
  71. arch_iommu = NULL;
  72. }
  73. EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch);
  74. /**
  75. * omap_iommu_save_ctx - Save registers for pm off-mode support
  76. * @obj: target iommu
  77. **/
  78. void omap_iommu_save_ctx(struct omap_iommu *obj)
  79. {
  80. arch_iommu->save_ctx(obj);
  81. }
  82. EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
  83. /**
  84. * omap_iommu_restore_ctx - Restore registers for pm off-mode support
  85. * @obj: target iommu
  86. **/
  87. void omap_iommu_restore_ctx(struct omap_iommu *obj)
  88. {
  89. arch_iommu->restore_ctx(obj);
  90. }
  91. EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
  92. /**
  93. * omap_iommu_arch_version - Return running iommu arch version
  94. **/
  95. u32 omap_iommu_arch_version(void)
  96. {
  97. return arch_iommu->version;
  98. }
  99. EXPORT_SYMBOL_GPL(omap_iommu_arch_version);
  100. static int iommu_enable(struct omap_iommu *obj)
  101. {
  102. int err;
  103. if (!obj)
  104. return -EINVAL;
  105. if (!arch_iommu)
  106. return -ENODEV;
  107. clk_enable(obj->clk);
  108. err = arch_iommu->enable(obj);
  109. clk_disable(obj->clk);
  110. return err;
  111. }
  112. static void iommu_disable(struct omap_iommu *obj)
  113. {
  114. if (!obj)
  115. return;
  116. clk_enable(obj->clk);
  117. arch_iommu->disable(obj);
  118. clk_disable(obj->clk);
  119. }
  120. /*
  121. * TLB operations
  122. */
  123. void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e)
  124. {
  125. BUG_ON(!cr || !e);
  126. arch_iommu->cr_to_e(cr, e);
  127. }
  128. EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e);
  129. static inline int iotlb_cr_valid(struct cr_regs *cr)
  130. {
  131. if (!cr)
  132. return -EINVAL;
  133. return arch_iommu->cr_valid(cr);
  134. }
  135. static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
  136. struct iotlb_entry *e)
  137. {
  138. if (!e)
  139. return NULL;
  140. return arch_iommu->alloc_cr(obj, e);
  141. }
  142. static u32 iotlb_cr_to_virt(struct cr_regs *cr)
  143. {
  144. return arch_iommu->cr_to_virt(cr);
  145. }
  146. static u32 get_iopte_attr(struct iotlb_entry *e)
  147. {
  148. return arch_iommu->get_pte_attr(e);
  149. }
  150. static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
  151. {
  152. return arch_iommu->fault_isr(obj, da);
  153. }
  154. static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
  155. {
  156. u32 val;
  157. val = iommu_read_reg(obj, MMU_LOCK);
  158. l->base = MMU_LOCK_BASE(val);
  159. l->vict = MMU_LOCK_VICT(val);
  160. }
  161. static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
  162. {
  163. u32 val;
  164. val = (l->base << MMU_LOCK_BASE_SHIFT);
  165. val |= (l->vict << MMU_LOCK_VICT_SHIFT);
  166. iommu_write_reg(obj, val, MMU_LOCK);
  167. }
  168. static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
  169. {
  170. arch_iommu->tlb_read_cr(obj, cr);
  171. }
  172. static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
  173. {
  174. arch_iommu->tlb_load_cr(obj, cr);
  175. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  176. iommu_write_reg(obj, 1, MMU_LD_TLB);
  177. }
  178. /**
  179. * iotlb_dump_cr - Dump an iommu tlb entry into buf
  180. * @obj: target iommu
  181. * @cr: contents of cam and ram register
  182. * @buf: output buffer
  183. **/
  184. static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,
  185. char *buf)
  186. {
  187. BUG_ON(!cr || !buf);
  188. return arch_iommu->dump_cr(obj, cr, buf);
  189. }
  190. /* only used in iotlb iteration for-loop */
  191. static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
  192. {
  193. struct cr_regs cr;
  194. struct iotlb_lock l;
  195. iotlb_lock_get(obj, &l);
  196. l.vict = n;
  197. iotlb_lock_set(obj, &l);
  198. iotlb_read_cr(obj, &cr);
  199. return cr;
  200. }
  201. /**
  202. * load_iotlb_entry - Set an iommu tlb entry
  203. * @obj: target iommu
  204. * @e: an iommu tlb entry info
  205. **/
  206. #ifdef PREFETCH_IOTLB
  207. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  208. {
  209. int err = 0;
  210. struct iotlb_lock l;
  211. struct cr_regs *cr;
  212. if (!obj || !obj->nr_tlb_entries || !e)
  213. return -EINVAL;
  214. clk_enable(obj->clk);
  215. iotlb_lock_get(obj, &l);
  216. if (l.base == obj->nr_tlb_entries) {
  217. dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
  218. err = -EBUSY;
  219. goto out;
  220. }
  221. if (!e->prsvd) {
  222. int i;
  223. struct cr_regs tmp;
  224. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
  225. if (!iotlb_cr_valid(&tmp))
  226. break;
  227. if (i == obj->nr_tlb_entries) {
  228. dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
  229. err = -EBUSY;
  230. goto out;
  231. }
  232. iotlb_lock_get(obj, &l);
  233. } else {
  234. l.vict = l.base;
  235. iotlb_lock_set(obj, &l);
  236. }
  237. cr = iotlb_alloc_cr(obj, e);
  238. if (IS_ERR(cr)) {
  239. clk_disable(obj->clk);
  240. return PTR_ERR(cr);
  241. }
  242. iotlb_load_cr(obj, cr);
  243. kfree(cr);
  244. if (e->prsvd)
  245. l.base++;
  246. /* increment victim for next tlb load */
  247. if (++l.vict == obj->nr_tlb_entries)
  248. l.vict = l.base;
  249. iotlb_lock_set(obj, &l);
  250. out:
  251. clk_disable(obj->clk);
  252. return err;
  253. }
  254. #else /* !PREFETCH_IOTLB */
  255. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  256. {
  257. return 0;
  258. }
  259. #endif /* !PREFETCH_IOTLB */
  260. static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  261. {
  262. return load_iotlb_entry(obj, e);
  263. }
  264. /**
  265. * flush_iotlb_page - Clear an iommu tlb entry
  266. * @obj: target iommu
  267. * @da: iommu device virtual address
  268. *
  269. * Clear an iommu tlb entry which includes 'da' address.
  270. **/
  271. static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
  272. {
  273. int i;
  274. struct cr_regs cr;
  275. clk_enable(obj->clk);
  276. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
  277. u32 start;
  278. size_t bytes;
  279. if (!iotlb_cr_valid(&cr))
  280. continue;
  281. start = iotlb_cr_to_virt(&cr);
  282. bytes = iopgsz_to_bytes(cr.cam & 3);
  283. if ((start <= da) && (da < start + bytes)) {
  284. dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
  285. __func__, start, da, bytes);
  286. iotlb_load_cr(obj, &cr);
  287. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  288. }
  289. }
  290. clk_disable(obj->clk);
  291. if (i == obj->nr_tlb_entries)
  292. dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
  293. }
  294. /**
  295. * flush_iotlb_all - Clear all iommu tlb entries
  296. * @obj: target iommu
  297. **/
  298. static void flush_iotlb_all(struct omap_iommu *obj)
  299. {
  300. struct iotlb_lock l;
  301. clk_enable(obj->clk);
  302. l.base = 0;
  303. l.vict = 0;
  304. iotlb_lock_set(obj, &l);
  305. iommu_write_reg(obj, 1, MMU_GFLUSH);
  306. clk_disable(obj->clk);
  307. }
  308. #if defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE)
  309. ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes)
  310. {
  311. if (!obj || !buf)
  312. return -EINVAL;
  313. clk_enable(obj->clk);
  314. bytes = arch_iommu->dump_ctx(obj, buf, bytes);
  315. clk_disable(obj->clk);
  316. return bytes;
  317. }
  318. EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx);
  319. static int
  320. __dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)
  321. {
  322. int i;
  323. struct iotlb_lock saved;
  324. struct cr_regs tmp;
  325. struct cr_regs *p = crs;
  326. clk_enable(obj->clk);
  327. iotlb_lock_get(obj, &saved);
  328. for_each_iotlb_cr(obj, num, i, tmp) {
  329. if (!iotlb_cr_valid(&tmp))
  330. continue;
  331. *p++ = tmp;
  332. }
  333. iotlb_lock_set(obj, &saved);
  334. clk_disable(obj->clk);
  335. return p - crs;
  336. }
  337. /**
  338. * omap_dump_tlb_entries - dump cr arrays to given buffer
  339. * @obj: target iommu
  340. * @buf: output buffer
  341. **/
  342. size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes)
  343. {
  344. int i, num;
  345. struct cr_regs *cr;
  346. char *p = buf;
  347. num = bytes / sizeof(*cr);
  348. num = min(obj->nr_tlb_entries, num);
  349. cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
  350. if (!cr)
  351. return 0;
  352. num = __dump_tlb_entries(obj, cr, num);
  353. for (i = 0; i < num; i++)
  354. p += iotlb_dump_cr(obj, cr + i, p);
  355. kfree(cr);
  356. return p - buf;
  357. }
  358. EXPORT_SYMBOL_GPL(omap_dump_tlb_entries);
  359. int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *))
  360. {
  361. return driver_for_each_device(&omap_iommu_driver.driver,
  362. NULL, data, fn);
  363. }
  364. EXPORT_SYMBOL_GPL(omap_foreach_iommu_device);
  365. #endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */
  366. /*
  367. * H/W pagetable operations
  368. */
  369. static void flush_iopgd_range(u32 *first, u32 *last)
  370. {
  371. /* FIXME: L2 cache should be taken care of if it exists */
  372. do {
  373. asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
  374. : : "r" (first));
  375. first += L1_CACHE_BYTES / sizeof(*first);
  376. } while (first <= last);
  377. }
  378. static void flush_iopte_range(u32 *first, u32 *last)
  379. {
  380. /* FIXME: L2 cache should be taken care of if it exists */
  381. do {
  382. asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
  383. : : "r" (first));
  384. first += L1_CACHE_BYTES / sizeof(*first);
  385. } while (first <= last);
  386. }
  387. static void iopte_free(u32 *iopte)
  388. {
  389. /* Note: freed iopte's must be clean ready for re-use */
  390. kmem_cache_free(iopte_cachep, iopte);
  391. }
  392. static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
  393. {
  394. u32 *iopte;
  395. /* a table has already existed */
  396. if (*iopgd)
  397. goto pte_ready;
  398. /*
  399. * do the allocation outside the page table lock
  400. */
  401. spin_unlock(&obj->page_table_lock);
  402. iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
  403. spin_lock(&obj->page_table_lock);
  404. if (!*iopgd) {
  405. if (!iopte)
  406. return ERR_PTR(-ENOMEM);
  407. *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
  408. flush_iopgd_range(iopgd, iopgd);
  409. dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
  410. } else {
  411. /* We raced, free the reduniovant table */
  412. iopte_free(iopte);
  413. }
  414. pte_ready:
  415. iopte = iopte_offset(iopgd, da);
  416. dev_vdbg(obj->dev,
  417. "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
  418. __func__, da, iopgd, *iopgd, iopte, *iopte);
  419. return iopte;
  420. }
  421. static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  422. {
  423. u32 *iopgd = iopgd_offset(obj, da);
  424. if ((da | pa) & ~IOSECTION_MASK) {
  425. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  426. __func__, da, pa, IOSECTION_SIZE);
  427. return -EINVAL;
  428. }
  429. *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
  430. flush_iopgd_range(iopgd, iopgd);
  431. return 0;
  432. }
  433. static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  434. {
  435. u32 *iopgd = iopgd_offset(obj, da);
  436. int i;
  437. if ((da | pa) & ~IOSUPER_MASK) {
  438. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  439. __func__, da, pa, IOSUPER_SIZE);
  440. return -EINVAL;
  441. }
  442. for (i = 0; i < 16; i++)
  443. *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
  444. flush_iopgd_range(iopgd, iopgd + 15);
  445. return 0;
  446. }
  447. static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  448. {
  449. u32 *iopgd = iopgd_offset(obj, da);
  450. u32 *iopte = iopte_alloc(obj, iopgd, da);
  451. if (IS_ERR(iopte))
  452. return PTR_ERR(iopte);
  453. *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
  454. flush_iopte_range(iopte, iopte);
  455. dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
  456. __func__, da, pa, iopte, *iopte);
  457. return 0;
  458. }
  459. static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  460. {
  461. u32 *iopgd = iopgd_offset(obj, da);
  462. u32 *iopte = iopte_alloc(obj, iopgd, da);
  463. int i;
  464. if ((da | pa) & ~IOLARGE_MASK) {
  465. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  466. __func__, da, pa, IOLARGE_SIZE);
  467. return -EINVAL;
  468. }
  469. if (IS_ERR(iopte))
  470. return PTR_ERR(iopte);
  471. for (i = 0; i < 16; i++)
  472. *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
  473. flush_iopte_range(iopte, iopte + 15);
  474. return 0;
  475. }
  476. static int
  477. iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
  478. {
  479. int (*fn)(struct omap_iommu *, u32, u32, u32);
  480. u32 prot;
  481. int err;
  482. if (!obj || !e)
  483. return -EINVAL;
  484. switch (e->pgsz) {
  485. case MMU_CAM_PGSZ_16M:
  486. fn = iopgd_alloc_super;
  487. break;
  488. case MMU_CAM_PGSZ_1M:
  489. fn = iopgd_alloc_section;
  490. break;
  491. case MMU_CAM_PGSZ_64K:
  492. fn = iopte_alloc_large;
  493. break;
  494. case MMU_CAM_PGSZ_4K:
  495. fn = iopte_alloc_page;
  496. break;
  497. default:
  498. fn = NULL;
  499. BUG();
  500. break;
  501. }
  502. prot = get_iopte_attr(e);
  503. spin_lock(&obj->page_table_lock);
  504. err = fn(obj, e->da, e->pa, prot);
  505. spin_unlock(&obj->page_table_lock);
  506. return err;
  507. }
  508. /**
  509. * omap_iopgtable_store_entry - Make an iommu pte entry
  510. * @obj: target iommu
  511. * @e: an iommu tlb entry info
  512. **/
  513. int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  514. {
  515. int err;
  516. flush_iotlb_page(obj, e->da);
  517. err = iopgtable_store_entry_core(obj, e);
  518. if (!err)
  519. prefetch_iotlb_entry(obj, e);
  520. return err;
  521. }
  522. EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry);
  523. /**
  524. * iopgtable_lookup_entry - Lookup an iommu pte entry
  525. * @obj: target iommu
  526. * @da: iommu device virtual address
  527. * @ppgd: iommu pgd entry pointer to be returned
  528. * @ppte: iommu pte entry pointer to be returned
  529. **/
  530. static void
  531. iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
  532. {
  533. u32 *iopgd, *iopte = NULL;
  534. iopgd = iopgd_offset(obj, da);
  535. if (!*iopgd)
  536. goto out;
  537. if (iopgd_is_table(*iopgd))
  538. iopte = iopte_offset(iopgd, da);
  539. out:
  540. *ppgd = iopgd;
  541. *ppte = iopte;
  542. }
  543. static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
  544. {
  545. size_t bytes;
  546. u32 *iopgd = iopgd_offset(obj, da);
  547. int nent = 1;
  548. if (!*iopgd)
  549. return 0;
  550. if (iopgd_is_table(*iopgd)) {
  551. int i;
  552. u32 *iopte = iopte_offset(iopgd, da);
  553. bytes = IOPTE_SIZE;
  554. if (*iopte & IOPTE_LARGE) {
  555. nent *= 16;
  556. /* rewind to the 1st entry */
  557. iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
  558. }
  559. bytes *= nent;
  560. memset(iopte, 0, nent * sizeof(*iopte));
  561. flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
  562. /*
  563. * do table walk to check if this table is necessary or not
  564. */
  565. iopte = iopte_offset(iopgd, 0);
  566. for (i = 0; i < PTRS_PER_IOPTE; i++)
  567. if (iopte[i])
  568. goto out;
  569. iopte_free(iopte);
  570. nent = 1; /* for the next L1 entry */
  571. } else {
  572. bytes = IOPGD_SIZE;
  573. if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
  574. nent *= 16;
  575. /* rewind to the 1st entry */
  576. iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
  577. }
  578. bytes *= nent;
  579. }
  580. memset(iopgd, 0, nent * sizeof(*iopgd));
  581. flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
  582. out:
  583. return bytes;
  584. }
  585. /**
  586. * iopgtable_clear_entry - Remove an iommu pte entry
  587. * @obj: target iommu
  588. * @da: iommu device virtual address
  589. **/
  590. static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
  591. {
  592. size_t bytes;
  593. spin_lock(&obj->page_table_lock);
  594. bytes = iopgtable_clear_entry_core(obj, da);
  595. flush_iotlb_page(obj, da);
  596. spin_unlock(&obj->page_table_lock);
  597. return bytes;
  598. }
  599. static void iopgtable_clear_entry_all(struct omap_iommu *obj)
  600. {
  601. int i;
  602. spin_lock(&obj->page_table_lock);
  603. for (i = 0; i < PTRS_PER_IOPGD; i++) {
  604. u32 da;
  605. u32 *iopgd;
  606. da = i << IOPGD_SHIFT;
  607. iopgd = iopgd_offset(obj, da);
  608. if (!*iopgd)
  609. continue;
  610. if (iopgd_is_table(*iopgd))
  611. iopte_free(iopte_offset(iopgd, 0));
  612. *iopgd = 0;
  613. flush_iopgd_range(iopgd, iopgd);
  614. }
  615. flush_iotlb_all(obj);
  616. spin_unlock(&obj->page_table_lock);
  617. }
  618. /*
  619. * Device IOMMU generic operations
  620. */
  621. static irqreturn_t iommu_fault_handler(int irq, void *data)
  622. {
  623. u32 da, errs;
  624. u32 *iopgd, *iopte;
  625. struct omap_iommu *obj = data;
  626. if (!obj->refcount)
  627. return IRQ_NONE;
  628. clk_enable(obj->clk);
  629. errs = iommu_report_fault(obj, &da);
  630. clk_disable(obj->clk);
  631. if (errs == 0)
  632. return IRQ_HANDLED;
  633. /* Fault callback or TLB/PTE Dynamic loading */
  634. if (obj->isr && !obj->isr(obj, da, errs, obj->isr_priv))
  635. return IRQ_HANDLED;
  636. iommu_disable(obj);
  637. iopgd = iopgd_offset(obj, da);
  638. if (!iopgd_is_table(*iopgd)) {
  639. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p "
  640. "*pgd:px%08x\n", obj->name, errs, da, iopgd, *iopgd);
  641. return IRQ_NONE;
  642. }
  643. iopte = iopte_offset(iopgd, da);
  644. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x "
  645. "pte:0x%p *pte:0x%08x\n", obj->name, errs, da, iopgd, *iopgd,
  646. iopte, *iopte);
  647. return IRQ_NONE;
  648. }
  649. static int device_match_by_alias(struct device *dev, void *data)
  650. {
  651. struct omap_iommu *obj = to_iommu(dev);
  652. const char *name = data;
  653. pr_debug("%s: %s %s\n", __func__, obj->name, name);
  654. return strcmp(obj->name, name) == 0;
  655. }
  656. /**
  657. * omap_find_iommu_device() - find an omap iommu device by name
  658. * @name: name of the iommu device
  659. *
  660. * The generic iommu API requires the caller to provide the device
  661. * he wishes to attach to a certain iommu domain.
  662. *
  663. * Drivers generally should not bother with this as it should just
  664. * be taken care of by the DMA-API using dev_archdata.
  665. *
  666. * This function is provided as an interim solution until the latter
  667. * materializes, and omap3isp is fully migrated to the DMA-API.
  668. */
  669. struct device *omap_find_iommu_device(const char *name)
  670. {
  671. return driver_find_device(&omap_iommu_driver.driver, NULL,
  672. (void *)name,
  673. device_match_by_alias);
  674. }
  675. EXPORT_SYMBOL_GPL(omap_find_iommu_device);
  676. /**
  677. * omap_iommu_attach() - attach iommu device to an iommu domain
  678. * @dev: target omap iommu device
  679. * @iopgd: page table
  680. **/
  681. static struct omap_iommu *omap_iommu_attach(struct device *dev, u32 *iopgd)
  682. {
  683. int err = -ENOMEM;
  684. struct omap_iommu *obj = to_iommu(dev);
  685. spin_lock(&obj->iommu_lock);
  686. /* an iommu device can only be attached once */
  687. if (++obj->refcount > 1) {
  688. dev_err(dev, "%s: already attached!\n", obj->name);
  689. err = -EBUSY;
  690. goto err_enable;
  691. }
  692. obj->iopgd = iopgd;
  693. err = iommu_enable(obj);
  694. if (err)
  695. goto err_enable;
  696. flush_iotlb_all(obj);
  697. if (!try_module_get(obj->owner))
  698. goto err_module;
  699. spin_unlock(&obj->iommu_lock);
  700. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  701. return obj;
  702. err_module:
  703. if (obj->refcount == 1)
  704. iommu_disable(obj);
  705. err_enable:
  706. obj->refcount--;
  707. spin_unlock(&obj->iommu_lock);
  708. return ERR_PTR(err);
  709. }
  710. /**
  711. * omap_iommu_detach - release iommu device
  712. * @obj: target iommu
  713. **/
  714. static void omap_iommu_detach(struct omap_iommu *obj)
  715. {
  716. if (!obj || IS_ERR(obj))
  717. return;
  718. spin_lock(&obj->iommu_lock);
  719. if (--obj->refcount == 0)
  720. iommu_disable(obj);
  721. module_put(obj->owner);
  722. obj->iopgd = NULL;
  723. spin_unlock(&obj->iommu_lock);
  724. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  725. }
  726. int omap_iommu_set_isr(const char *name,
  727. int (*isr)(struct omap_iommu *obj, u32 da, u32 iommu_errs,
  728. void *priv),
  729. void *isr_priv)
  730. {
  731. struct device *dev;
  732. struct omap_iommu *obj;
  733. dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
  734. device_match_by_alias);
  735. if (!dev)
  736. return -ENODEV;
  737. obj = to_iommu(dev);
  738. mutex_lock(&obj->iommu_lock);
  739. if (obj->refcount != 0) {
  740. mutex_unlock(&obj->iommu_lock);
  741. return -EBUSY;
  742. }
  743. obj->isr = isr;
  744. obj->isr_priv = isr_priv;
  745. mutex_unlock(&obj->iommu_lock);
  746. return 0;
  747. }
  748. EXPORT_SYMBOL_GPL(omap_iommu_set_isr);
  749. /*
  750. * OMAP Device MMU(IOMMU) detection
  751. */
  752. static int __devinit omap_iommu_probe(struct platform_device *pdev)
  753. {
  754. int err = -ENODEV;
  755. int irq;
  756. struct omap_iommu *obj;
  757. struct resource *res;
  758. struct iommu_platform_data *pdata = pdev->dev.platform_data;
  759. if (pdev->num_resources != 2)
  760. return -EINVAL;
  761. obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
  762. if (!obj)
  763. return -ENOMEM;
  764. obj->clk = clk_get(&pdev->dev, pdata->clk_name);
  765. if (IS_ERR(obj->clk))
  766. goto err_clk;
  767. obj->nr_tlb_entries = pdata->nr_tlb_entries;
  768. obj->name = pdata->name;
  769. obj->dev = &pdev->dev;
  770. obj->ctx = (void *)obj + sizeof(*obj);
  771. obj->da_start = pdata->da_start;
  772. obj->da_end = pdata->da_end;
  773. spin_lock_init(&obj->iommu_lock);
  774. mutex_init(&obj->mmap_lock);
  775. spin_lock_init(&obj->page_table_lock);
  776. INIT_LIST_HEAD(&obj->mmap);
  777. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  778. if (!res) {
  779. err = -ENODEV;
  780. goto err_mem;
  781. }
  782. res = request_mem_region(res->start, resource_size(res),
  783. dev_name(&pdev->dev));
  784. if (!res) {
  785. err = -EIO;
  786. goto err_mem;
  787. }
  788. obj->regbase = ioremap(res->start, resource_size(res));
  789. if (!obj->regbase) {
  790. err = -ENOMEM;
  791. goto err_ioremap;
  792. }
  793. irq = platform_get_irq(pdev, 0);
  794. if (irq < 0) {
  795. err = -ENODEV;
  796. goto err_irq;
  797. }
  798. err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
  799. dev_name(&pdev->dev), obj);
  800. if (err < 0)
  801. goto err_irq;
  802. platform_set_drvdata(pdev, obj);
  803. dev_info(&pdev->dev, "%s registered\n", obj->name);
  804. return 0;
  805. err_irq:
  806. iounmap(obj->regbase);
  807. err_ioremap:
  808. release_mem_region(res->start, resource_size(res));
  809. err_mem:
  810. clk_put(obj->clk);
  811. err_clk:
  812. kfree(obj);
  813. return err;
  814. }
  815. static int __devexit omap_iommu_remove(struct platform_device *pdev)
  816. {
  817. int irq;
  818. struct resource *res;
  819. struct omap_iommu *obj = platform_get_drvdata(pdev);
  820. platform_set_drvdata(pdev, NULL);
  821. iopgtable_clear_entry_all(obj);
  822. irq = platform_get_irq(pdev, 0);
  823. free_irq(irq, obj);
  824. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  825. release_mem_region(res->start, resource_size(res));
  826. iounmap(obj->regbase);
  827. clk_put(obj->clk);
  828. dev_info(&pdev->dev, "%s removed\n", obj->name);
  829. kfree(obj);
  830. return 0;
  831. }
  832. static struct platform_driver omap_iommu_driver = {
  833. .probe = omap_iommu_probe,
  834. .remove = __devexit_p(omap_iommu_remove),
  835. .driver = {
  836. .name = "omap-iommu",
  837. },
  838. };
  839. static void iopte_cachep_ctor(void *iopte)
  840. {
  841. clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
  842. }
  843. static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
  844. phys_addr_t pa, int order, int prot)
  845. {
  846. struct omap_iommu_domain *omap_domain = domain->priv;
  847. struct omap_iommu *oiommu = omap_domain->iommu_dev;
  848. struct device *dev = oiommu->dev;
  849. size_t bytes = PAGE_SIZE << order;
  850. struct iotlb_entry e;
  851. int omap_pgsz;
  852. u32 ret, flags;
  853. /* we only support mapping a single iommu page for now */
  854. omap_pgsz = bytes_to_iopgsz(bytes);
  855. if (omap_pgsz < 0) {
  856. dev_err(dev, "invalid size to map: %d\n", bytes);
  857. return -EINVAL;
  858. }
  859. dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
  860. flags = omap_pgsz | prot;
  861. iotlb_init_entry(&e, da, pa, flags);
  862. ret = omap_iopgtable_store_entry(oiommu, &e);
  863. if (ret) {
  864. dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret);
  865. return ret;
  866. }
  867. return 0;
  868. }
  869. static int omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
  870. int order)
  871. {
  872. struct omap_iommu_domain *omap_domain = domain->priv;
  873. struct omap_iommu *oiommu = omap_domain->iommu_dev;
  874. struct device *dev = oiommu->dev;
  875. size_t bytes = PAGE_SIZE << order;
  876. size_t ret;
  877. dev_dbg(dev, "unmapping da 0x%lx size 0x%x\n", da, bytes);
  878. ret = iopgtable_clear_entry(oiommu, da);
  879. if (ret != bytes) {
  880. dev_err(dev, "entry @ 0x%lx was %d; not %d\n", da, ret, bytes);
  881. return -EINVAL;
  882. }
  883. return 0;
  884. }
  885. static int
  886. omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
  887. {
  888. struct omap_iommu_domain *omap_domain = domain->priv;
  889. struct omap_iommu *oiommu;
  890. int ret = 0;
  891. spin_lock(&omap_domain->lock);
  892. /* only a single device is supported per domain for now */
  893. if (omap_domain->iommu_dev) {
  894. dev_err(dev, "iommu domain is already attached\n");
  895. ret = -EBUSY;
  896. goto out;
  897. }
  898. /* get a handle to and enable the omap iommu */
  899. oiommu = omap_iommu_attach(dev, omap_domain->pgtable);
  900. if (IS_ERR(oiommu)) {
  901. ret = PTR_ERR(oiommu);
  902. dev_err(dev, "can't get omap iommu: %d\n", ret);
  903. goto out;
  904. }
  905. omap_domain->iommu_dev = oiommu;
  906. out:
  907. spin_unlock(&omap_domain->lock);
  908. return ret;
  909. }
  910. static void omap_iommu_detach_dev(struct iommu_domain *domain,
  911. struct device *dev)
  912. {
  913. struct omap_iommu_domain *omap_domain = domain->priv;
  914. struct omap_iommu *oiommu = to_iommu(dev);
  915. spin_lock(&omap_domain->lock);
  916. /* only a single device is supported per domain for now */
  917. if (omap_domain->iommu_dev != oiommu) {
  918. dev_err(dev, "invalid iommu device\n");
  919. goto out;
  920. }
  921. iopgtable_clear_entry_all(oiommu);
  922. omap_iommu_detach(oiommu);
  923. omap_domain->iommu_dev = NULL;
  924. out:
  925. spin_unlock(&omap_domain->lock);
  926. }
  927. static int omap_iommu_domain_init(struct iommu_domain *domain)
  928. {
  929. struct omap_iommu_domain *omap_domain;
  930. omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
  931. if (!omap_domain) {
  932. pr_err("kzalloc failed\n");
  933. goto out;
  934. }
  935. omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
  936. if (!omap_domain->pgtable) {
  937. pr_err("kzalloc failed\n");
  938. goto fail_nomem;
  939. }
  940. /*
  941. * should never fail, but please keep this around to ensure
  942. * we keep the hardware happy
  943. */
  944. BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE));
  945. clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
  946. spin_lock_init(&omap_domain->lock);
  947. domain->priv = omap_domain;
  948. return 0;
  949. fail_nomem:
  950. kfree(omap_domain);
  951. out:
  952. return -ENOMEM;
  953. }
  954. /* assume device was already detached */
  955. static void omap_iommu_domain_destroy(struct iommu_domain *domain)
  956. {
  957. struct omap_iommu_domain *omap_domain = domain->priv;
  958. domain->priv = NULL;
  959. kfree(omap_domain->pgtable);
  960. kfree(omap_domain);
  961. }
  962. static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
  963. unsigned long da)
  964. {
  965. struct omap_iommu_domain *omap_domain = domain->priv;
  966. struct omap_iommu *oiommu = omap_domain->iommu_dev;
  967. struct device *dev = oiommu->dev;
  968. u32 *pgd, *pte;
  969. phys_addr_t ret = 0;
  970. iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
  971. if (pte) {
  972. if (iopte_is_small(*pte))
  973. ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
  974. else if (iopte_is_large(*pte))
  975. ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
  976. else
  977. dev_err(dev, "bogus pte 0x%x", *pte);
  978. } else {
  979. if (iopgd_is_section(*pgd))
  980. ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
  981. else if (iopgd_is_super(*pgd))
  982. ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
  983. else
  984. dev_err(dev, "bogus pgd 0x%x", *pgd);
  985. }
  986. return ret;
  987. }
  988. static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
  989. unsigned long cap)
  990. {
  991. return 0;
  992. }
  993. static struct iommu_ops omap_iommu_ops = {
  994. .domain_init = omap_iommu_domain_init,
  995. .domain_destroy = omap_iommu_domain_destroy,
  996. .attach_dev = omap_iommu_attach_dev,
  997. .detach_dev = omap_iommu_detach_dev,
  998. .map = omap_iommu_map,
  999. .unmap = omap_iommu_unmap,
  1000. .iova_to_phys = omap_iommu_iova_to_phys,
  1001. .domain_has_cap = omap_iommu_domain_has_cap,
  1002. };
  1003. static int __init omap_iommu_init(void)
  1004. {
  1005. struct kmem_cache *p;
  1006. const unsigned long flags = SLAB_HWCACHE_ALIGN;
  1007. size_t align = 1 << 10; /* L2 pagetable alignement */
  1008. p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
  1009. iopte_cachep_ctor);
  1010. if (!p)
  1011. return -ENOMEM;
  1012. iopte_cachep = p;
  1013. register_iommu(&omap_iommu_ops);
  1014. return platform_driver_register(&omap_iommu_driver);
  1015. }
  1016. module_init(omap_iommu_init);
  1017. static void __exit omap_iommu_exit(void)
  1018. {
  1019. kmem_cache_destroy(iopte_cachep);
  1020. platform_driver_unregister(&omap_iommu_driver);
  1021. }
  1022. module_exit(omap_iommu_exit);
  1023. MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
  1024. MODULE_ALIAS("platform:omap-iommu");
  1025. MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
  1026. MODULE_LICENSE("GPL v2");