dma-mapping.c 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822
  1. /*
  2. * linux/arch/arm/mm/dma-mapping.c
  3. *
  4. * Copyright (C) 2000-2004 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. *
  10. * DMA uncached mapping support.
  11. */
  12. #include <linux/module.h>
  13. #include <linux/mm.h>
  14. #include <linux/gfp.h>
  15. #include <linux/errno.h>
  16. #include <linux/list.h>
  17. #include <linux/init.h>
  18. #include <linux/device.h>
  19. #include <linux/dma-mapping.h>
  20. #include <linux/dma-contiguous.h>
  21. #include <linux/highmem.h>
  22. #include <linux/memblock.h>
  23. #include <linux/slab.h>
  24. #include <linux/iommu.h>
  25. #include <linux/io.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/sizes.h>
  28. #include <asm/memory.h>
  29. #include <asm/highmem.h>
  30. #include <asm/cacheflush.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/mach/arch.h>
  33. #include <asm/dma-iommu.h>
  34. #include <asm/mach/map.h>
  35. #include <asm/system_info.h>
  36. #include <asm/dma-contiguous.h>
  37. #include "mm.h"
  38. /*
  39. * The DMA API is built upon the notion of "buffer ownership". A buffer
  40. * is either exclusively owned by the CPU (and therefore may be accessed
  41. * by it) or exclusively owned by the DMA device. These helper functions
  42. * represent the transitions between these two ownership states.
  43. *
  44. * Note, however, that on later ARMs, this notion does not work due to
  45. * speculative prefetches. We model our approach on the assumption that
  46. * the CPU does do speculative prefetches, which means we clean caches
  47. * before transfers and delay cache invalidation until transfer completion.
  48. *
  49. */
  50. static void __dma_page_cpu_to_dev(struct page *, unsigned long,
  51. size_t, enum dma_data_direction);
  52. static void __dma_page_dev_to_cpu(struct page *, unsigned long,
  53. size_t, enum dma_data_direction);
  54. /**
  55. * arm_dma_map_page - map a portion of a page for streaming DMA
  56. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  57. * @page: page that buffer resides in
  58. * @offset: offset into page for start of buffer
  59. * @size: size of buffer to map
  60. * @dir: DMA transfer direction
  61. *
  62. * Ensure that any data held in the cache is appropriately discarded
  63. * or written back.
  64. *
  65. * The device owns this memory once this call has completed. The CPU
  66. * can regain ownership by calling dma_unmap_page().
  67. */
  68. static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
  69. unsigned long offset, size_t size, enum dma_data_direction dir,
  70. struct dma_attrs *attrs)
  71. {
  72. if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  73. __dma_page_cpu_to_dev(page, offset, size, dir);
  74. return pfn_to_dma(dev, page_to_pfn(page)) + offset;
  75. }
  76. static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
  77. unsigned long offset, size_t size, enum dma_data_direction dir,
  78. struct dma_attrs *attrs)
  79. {
  80. return pfn_to_dma(dev, page_to_pfn(page)) + offset;
  81. }
  82. /**
  83. * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
  84. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  85. * @handle: DMA address of buffer
  86. * @size: size of buffer (same as passed to dma_map_page)
  87. * @dir: DMA transfer direction (same as passed to dma_map_page)
  88. *
  89. * Unmap a page streaming mode DMA translation. The handle and size
  90. * must match what was provided in the previous dma_map_page() call.
  91. * All other usages are undefined.
  92. *
  93. * After this call, reads by the CPU to the buffer are guaranteed to see
  94. * whatever the device wrote there.
  95. */
  96. static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
  97. size_t size, enum dma_data_direction dir,
  98. struct dma_attrs *attrs)
  99. {
  100. if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  101. __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
  102. handle & ~PAGE_MASK, size, dir);
  103. }
  104. static void arm_dma_sync_single_for_cpu(struct device *dev,
  105. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  106. {
  107. unsigned int offset = handle & (PAGE_SIZE - 1);
  108. struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
  109. __dma_page_dev_to_cpu(page, offset, size, dir);
  110. }
  111. static void arm_dma_sync_single_for_device(struct device *dev,
  112. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  113. {
  114. unsigned int offset = handle & (PAGE_SIZE - 1);
  115. struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
  116. __dma_page_cpu_to_dev(page, offset, size, dir);
  117. }
  118. static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
  119. struct dma_map_ops arm_dma_ops = {
  120. .alloc = arm_dma_alloc,
  121. .free = arm_dma_free,
  122. .mmap = arm_dma_mmap,
  123. .get_sgtable = arm_dma_get_sgtable,
  124. .map_page = arm_dma_map_page,
  125. .unmap_page = arm_dma_unmap_page,
  126. .map_sg = arm_dma_map_sg,
  127. .unmap_sg = arm_dma_unmap_sg,
  128. .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
  129. .sync_single_for_device = arm_dma_sync_single_for_device,
  130. .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
  131. .sync_sg_for_device = arm_dma_sync_sg_for_device,
  132. .set_dma_mask = arm_dma_set_mask,
  133. };
  134. EXPORT_SYMBOL(arm_dma_ops);
  135. static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
  136. dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
  137. static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
  138. dma_addr_t handle, struct dma_attrs *attrs);
  139. struct dma_map_ops arm_coherent_dma_ops = {
  140. .alloc = arm_coherent_dma_alloc,
  141. .free = arm_coherent_dma_free,
  142. .mmap = arm_dma_mmap,
  143. .get_sgtable = arm_dma_get_sgtable,
  144. .map_page = arm_coherent_dma_map_page,
  145. .map_sg = arm_dma_map_sg,
  146. .set_dma_mask = arm_dma_set_mask,
  147. };
  148. EXPORT_SYMBOL(arm_coherent_dma_ops);
  149. static u64 get_coherent_dma_mask(struct device *dev)
  150. {
  151. u64 mask = (u64)arm_dma_limit;
  152. if (dev) {
  153. mask = dev->coherent_dma_mask;
  154. /*
  155. * Sanity check the DMA mask - it must be non-zero, and
  156. * must be able to be satisfied by a DMA allocation.
  157. */
  158. if (mask == 0) {
  159. dev_warn(dev, "coherent DMA mask is unset\n");
  160. return 0;
  161. }
  162. if ((~mask) & (u64)arm_dma_limit) {
  163. dev_warn(dev, "coherent DMA mask %#llx is smaller "
  164. "than system GFP_DMA mask %#llx\n",
  165. mask, (u64)arm_dma_limit);
  166. return 0;
  167. }
  168. }
  169. return mask;
  170. }
  171. static void __dma_clear_buffer(struct page *page, size_t size)
  172. {
  173. void *ptr;
  174. /*
  175. * Ensure that the allocated pages are zeroed, and that any data
  176. * lurking in the kernel direct-mapped region is invalidated.
  177. */
  178. ptr = page_address(page);
  179. if (ptr) {
  180. memset(ptr, 0, size);
  181. dmac_flush_range(ptr, ptr + size);
  182. outer_flush_range(__pa(ptr), __pa(ptr) + size);
  183. }
  184. }
  185. /*
  186. * Allocate a DMA buffer for 'dev' of size 'size' using the
  187. * specified gfp mask. Note that 'size' must be page aligned.
  188. */
  189. static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
  190. {
  191. unsigned long order = get_order(size);
  192. struct page *page, *p, *e;
  193. page = alloc_pages(gfp, order);
  194. if (!page)
  195. return NULL;
  196. /*
  197. * Now split the huge page and free the excess pages
  198. */
  199. split_page(page, order);
  200. for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
  201. __free_page(p);
  202. __dma_clear_buffer(page, size);
  203. return page;
  204. }
  205. /*
  206. * Free a DMA buffer. 'size' must be page aligned.
  207. */
  208. static void __dma_free_buffer(struct page *page, size_t size)
  209. {
  210. struct page *e = page + (size >> PAGE_SHIFT);
  211. while (page < e) {
  212. __free_page(page);
  213. page++;
  214. }
  215. }
  216. #ifdef CONFIG_MMU
  217. #ifdef CONFIG_HUGETLB_PAGE
  218. #error ARM Coherent DMA allocator does not (yet) support huge TLB
  219. #endif
  220. static void *__alloc_from_contiguous(struct device *dev, size_t size,
  221. pgprot_t prot, struct page **ret_page);
  222. static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
  223. pgprot_t prot, struct page **ret_page,
  224. const void *caller);
  225. static void *
  226. __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
  227. const void *caller)
  228. {
  229. struct vm_struct *area;
  230. unsigned long addr;
  231. /*
  232. * DMA allocation can be mapped to user space, so lets
  233. * set VM_USERMAP flags too.
  234. */
  235. area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
  236. caller);
  237. if (!area)
  238. return NULL;
  239. addr = (unsigned long)area->addr;
  240. area->phys_addr = __pfn_to_phys(page_to_pfn(page));
  241. if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
  242. vunmap((void *)addr);
  243. return NULL;
  244. }
  245. return (void *)addr;
  246. }
  247. static void __dma_free_remap(void *cpu_addr, size_t size)
  248. {
  249. unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
  250. struct vm_struct *area = find_vm_area(cpu_addr);
  251. if (!area || (area->flags & flags) != flags) {
  252. WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
  253. return;
  254. }
  255. unmap_kernel_range((unsigned long)cpu_addr, size);
  256. vunmap(cpu_addr);
  257. }
  258. #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
  259. struct dma_pool {
  260. size_t size;
  261. spinlock_t lock;
  262. unsigned long *bitmap;
  263. unsigned long nr_pages;
  264. void *vaddr;
  265. struct page **pages;
  266. };
  267. static struct dma_pool atomic_pool = {
  268. .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
  269. };
  270. static int __init early_coherent_pool(char *p)
  271. {
  272. atomic_pool.size = memparse(p, &p);
  273. return 0;
  274. }
  275. early_param("coherent_pool", early_coherent_pool);
  276. void __init init_dma_coherent_pool_size(unsigned long size)
  277. {
  278. /*
  279. * Catch any attempt to set the pool size too late.
  280. */
  281. BUG_ON(atomic_pool.vaddr);
  282. /*
  283. * Set architecture specific coherent pool size only if
  284. * it has not been changed by kernel command line parameter.
  285. */
  286. if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
  287. atomic_pool.size = size;
  288. }
  289. /*
  290. * Initialise the coherent pool for atomic allocations.
  291. */
  292. static int __init atomic_pool_init(void)
  293. {
  294. struct dma_pool *pool = &atomic_pool;
  295. pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
  296. unsigned long nr_pages = pool->size >> PAGE_SHIFT;
  297. unsigned long *bitmap;
  298. struct page *page;
  299. struct page **pages;
  300. void *ptr;
  301. int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
  302. bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  303. if (!bitmap)
  304. goto no_bitmap;
  305. pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
  306. if (!pages)
  307. goto no_pages;
  308. if (IS_ENABLED(CONFIG_CMA))
  309. ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
  310. else
  311. ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot,
  312. &page, NULL);
  313. if (ptr) {
  314. int i;
  315. for (i = 0; i < nr_pages; i++)
  316. pages[i] = page + i;
  317. spin_lock_init(&pool->lock);
  318. pool->vaddr = ptr;
  319. pool->pages = pages;
  320. pool->bitmap = bitmap;
  321. pool->nr_pages = nr_pages;
  322. pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
  323. (unsigned)pool->size / 1024);
  324. return 0;
  325. }
  326. kfree(pages);
  327. no_pages:
  328. kfree(bitmap);
  329. no_bitmap:
  330. pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
  331. (unsigned)pool->size / 1024);
  332. return -ENOMEM;
  333. }
  334. /*
  335. * CMA is activated by core_initcall, so we must be called after it.
  336. */
  337. postcore_initcall(atomic_pool_init);
  338. struct dma_contig_early_reserve {
  339. phys_addr_t base;
  340. unsigned long size;
  341. };
  342. static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
  343. static int dma_mmu_remap_num __initdata;
  344. void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
  345. {
  346. dma_mmu_remap[dma_mmu_remap_num].base = base;
  347. dma_mmu_remap[dma_mmu_remap_num].size = size;
  348. dma_mmu_remap_num++;
  349. }
  350. void __init dma_contiguous_remap(void)
  351. {
  352. int i;
  353. for (i = 0; i < dma_mmu_remap_num; i++) {
  354. phys_addr_t start = dma_mmu_remap[i].base;
  355. phys_addr_t end = start + dma_mmu_remap[i].size;
  356. struct map_desc map;
  357. unsigned long addr;
  358. if (end > arm_lowmem_limit)
  359. end = arm_lowmem_limit;
  360. if (start >= end)
  361. continue;
  362. map.pfn = __phys_to_pfn(start);
  363. map.virtual = __phys_to_virt(start);
  364. map.length = end - start;
  365. map.type = MT_MEMORY_DMA_READY;
  366. /*
  367. * Clear previous low-memory mapping
  368. */
  369. for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
  370. addr += PMD_SIZE)
  371. pmd_clear(pmd_off_k(addr));
  372. iotable_init(&map, 1);
  373. }
  374. }
  375. static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
  376. void *data)
  377. {
  378. struct page *page = virt_to_page(addr);
  379. pgprot_t prot = *(pgprot_t *)data;
  380. set_pte_ext(pte, mk_pte(page, prot), 0);
  381. return 0;
  382. }
  383. static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
  384. {
  385. unsigned long start = (unsigned long) page_address(page);
  386. unsigned end = start + size;
  387. apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
  388. dsb();
  389. flush_tlb_kernel_range(start, end);
  390. }
  391. static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
  392. pgprot_t prot, struct page **ret_page,
  393. const void *caller)
  394. {
  395. struct page *page;
  396. void *ptr;
  397. page = __dma_alloc_buffer(dev, size, gfp);
  398. if (!page)
  399. return NULL;
  400. ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
  401. if (!ptr) {
  402. __dma_free_buffer(page, size);
  403. return NULL;
  404. }
  405. *ret_page = page;
  406. return ptr;
  407. }
  408. static void *__alloc_from_pool(size_t size, struct page **ret_page)
  409. {
  410. struct dma_pool *pool = &atomic_pool;
  411. unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  412. unsigned int pageno;
  413. unsigned long flags;
  414. void *ptr = NULL;
  415. unsigned long align_mask;
  416. if (!pool->vaddr) {
  417. WARN(1, "coherent pool not initialised!\n");
  418. return NULL;
  419. }
  420. /*
  421. * Align the region allocation - allocations from pool are rather
  422. * small, so align them to their order in pages, minimum is a page
  423. * size. This helps reduce fragmentation of the DMA space.
  424. */
  425. align_mask = (1 << get_order(size)) - 1;
  426. spin_lock_irqsave(&pool->lock, flags);
  427. pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
  428. 0, count, align_mask);
  429. if (pageno < pool->nr_pages) {
  430. bitmap_set(pool->bitmap, pageno, count);
  431. ptr = pool->vaddr + PAGE_SIZE * pageno;
  432. *ret_page = pool->pages[pageno];
  433. } else {
  434. pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
  435. "Please increase it with coherent_pool= kernel parameter!\n",
  436. (unsigned)pool->size / 1024);
  437. }
  438. spin_unlock_irqrestore(&pool->lock, flags);
  439. return ptr;
  440. }
  441. static bool __in_atomic_pool(void *start, size_t size)
  442. {
  443. struct dma_pool *pool = &atomic_pool;
  444. void *end = start + size;
  445. void *pool_start = pool->vaddr;
  446. void *pool_end = pool->vaddr + pool->size;
  447. if (start < pool_start || start >= pool_end)
  448. return false;
  449. if (end <= pool_end)
  450. return true;
  451. WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
  452. start, end - 1, pool_start, pool_end - 1);
  453. return false;
  454. }
  455. static int __free_from_pool(void *start, size_t size)
  456. {
  457. struct dma_pool *pool = &atomic_pool;
  458. unsigned long pageno, count;
  459. unsigned long flags;
  460. if (!__in_atomic_pool(start, size))
  461. return 0;
  462. pageno = (start - pool->vaddr) >> PAGE_SHIFT;
  463. count = size >> PAGE_SHIFT;
  464. spin_lock_irqsave(&pool->lock, flags);
  465. bitmap_clear(pool->bitmap, pageno, count);
  466. spin_unlock_irqrestore(&pool->lock, flags);
  467. return 1;
  468. }
  469. static void *__alloc_from_contiguous(struct device *dev, size_t size,
  470. pgprot_t prot, struct page **ret_page)
  471. {
  472. unsigned long order = get_order(size);
  473. size_t count = size >> PAGE_SHIFT;
  474. struct page *page;
  475. page = dma_alloc_from_contiguous(dev, count, order);
  476. if (!page)
  477. return NULL;
  478. __dma_clear_buffer(page, size);
  479. __dma_remap(page, size, prot);
  480. *ret_page = page;
  481. return page_address(page);
  482. }
  483. static void __free_from_contiguous(struct device *dev, struct page *page,
  484. size_t size)
  485. {
  486. __dma_remap(page, size, pgprot_kernel);
  487. dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
  488. }
  489. static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
  490. {
  491. prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
  492. pgprot_writecombine(prot) :
  493. pgprot_dmacoherent(prot);
  494. return prot;
  495. }
  496. #define nommu() 0
  497. #else /* !CONFIG_MMU */
  498. #define nommu() 1
  499. #define __get_dma_pgprot(attrs, prot) __pgprot(0)
  500. #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
  501. #define __alloc_from_pool(size, ret_page) NULL
  502. #define __alloc_from_contiguous(dev, size, prot, ret) NULL
  503. #define __free_from_pool(cpu_addr, size) 0
  504. #define __free_from_contiguous(dev, page, size) do { } while (0)
  505. #define __dma_free_remap(cpu_addr, size) do { } while (0)
  506. #endif /* CONFIG_MMU */
  507. static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
  508. struct page **ret_page)
  509. {
  510. struct page *page;
  511. page = __dma_alloc_buffer(dev, size, gfp);
  512. if (!page)
  513. return NULL;
  514. *ret_page = page;
  515. return page_address(page);
  516. }
  517. static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
  518. gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
  519. {
  520. u64 mask = get_coherent_dma_mask(dev);
  521. struct page *page;
  522. void *addr;
  523. #ifdef CONFIG_DMA_API_DEBUG
  524. u64 limit = (mask + 1) & ~mask;
  525. if (limit && size >= limit) {
  526. dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
  527. size, mask);
  528. return NULL;
  529. }
  530. #endif
  531. if (!mask)
  532. return NULL;
  533. if (mask < 0xffffffffULL)
  534. gfp |= GFP_DMA;
  535. /*
  536. * Following is a work-around (a.k.a. hack) to prevent pages
  537. * with __GFP_COMP being passed to split_page() which cannot
  538. * handle them. The real problem is that this flag probably
  539. * should be 0 on ARM as it is not supported on this
  540. * platform; see CONFIG_HUGETLBFS.
  541. */
  542. gfp &= ~(__GFP_COMP);
  543. *handle = DMA_ERROR_CODE;
  544. size = PAGE_ALIGN(size);
  545. if (is_coherent || nommu())
  546. addr = __alloc_simple_buffer(dev, size, gfp, &page);
  547. else if (gfp & GFP_ATOMIC)
  548. addr = __alloc_from_pool(size, &page);
  549. else if (!IS_ENABLED(CONFIG_CMA))
  550. addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
  551. else
  552. addr = __alloc_from_contiguous(dev, size, prot, &page);
  553. if (addr)
  554. *handle = pfn_to_dma(dev, page_to_pfn(page));
  555. return addr;
  556. }
  557. /*
  558. * Allocate DMA-coherent memory space and return both the kernel remapped
  559. * virtual and bus address for that space.
  560. */
  561. void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
  562. gfp_t gfp, struct dma_attrs *attrs)
  563. {
  564. pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
  565. void *memory;
  566. if (dma_alloc_from_coherent(dev, size, handle, &memory))
  567. return memory;
  568. return __dma_alloc(dev, size, handle, gfp, prot, false,
  569. __builtin_return_address(0));
  570. }
  571. static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
  572. dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
  573. {
  574. pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
  575. void *memory;
  576. if (dma_alloc_from_coherent(dev, size, handle, &memory))
  577. return memory;
  578. return __dma_alloc(dev, size, handle, gfp, prot, true,
  579. __builtin_return_address(0));
  580. }
  581. /*
  582. * Create userspace mapping for the DMA-coherent memory.
  583. */
  584. int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
  585. void *cpu_addr, dma_addr_t dma_addr, size_t size,
  586. struct dma_attrs *attrs)
  587. {
  588. int ret = -ENXIO;
  589. #ifdef CONFIG_MMU
  590. unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  591. unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
  592. unsigned long pfn = dma_to_pfn(dev, dma_addr);
  593. unsigned long off = vma->vm_pgoff;
  594. vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
  595. if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
  596. return ret;
  597. if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
  598. ret = remap_pfn_range(vma, vma->vm_start,
  599. pfn + off,
  600. vma->vm_end - vma->vm_start,
  601. vma->vm_page_prot);
  602. }
  603. #endif /* CONFIG_MMU */
  604. return ret;
  605. }
  606. /*
  607. * Free a buffer as defined by the above mapping.
  608. */
  609. static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
  610. dma_addr_t handle, struct dma_attrs *attrs,
  611. bool is_coherent)
  612. {
  613. struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
  614. if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
  615. return;
  616. size = PAGE_ALIGN(size);
  617. if (is_coherent || nommu()) {
  618. __dma_free_buffer(page, size);
  619. } else if (__free_from_pool(cpu_addr, size)) {
  620. return;
  621. } else if (!IS_ENABLED(CONFIG_CMA)) {
  622. __dma_free_remap(cpu_addr, size);
  623. __dma_free_buffer(page, size);
  624. } else {
  625. /*
  626. * Non-atomic allocations cannot be freed with IRQs disabled
  627. */
  628. WARN_ON(irqs_disabled());
  629. __free_from_contiguous(dev, page, size);
  630. }
  631. }
  632. void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
  633. dma_addr_t handle, struct dma_attrs *attrs)
  634. {
  635. __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
  636. }
  637. static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
  638. dma_addr_t handle, struct dma_attrs *attrs)
  639. {
  640. __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
  641. }
  642. int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
  643. void *cpu_addr, dma_addr_t handle, size_t size,
  644. struct dma_attrs *attrs)
  645. {
  646. struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
  647. int ret;
  648. ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
  649. if (unlikely(ret))
  650. return ret;
  651. sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
  652. return 0;
  653. }
  654. static void dma_cache_maint_page(struct page *page, unsigned long offset,
  655. size_t size, enum dma_data_direction dir,
  656. void (*op)(const void *, size_t, int))
  657. {
  658. /*
  659. * A single sg entry may refer to multiple physically contiguous
  660. * pages. But we still need to process highmem pages individually.
  661. * If highmem is not configured then the bulk of this loop gets
  662. * optimized out.
  663. */
  664. size_t left = size;
  665. do {
  666. size_t len = left;
  667. void *vaddr;
  668. if (PageHighMem(page)) {
  669. if (len + offset > PAGE_SIZE) {
  670. if (offset >= PAGE_SIZE) {
  671. page += offset / PAGE_SIZE;
  672. offset %= PAGE_SIZE;
  673. }
  674. len = PAGE_SIZE - offset;
  675. }
  676. vaddr = kmap_high_get(page);
  677. if (vaddr) {
  678. vaddr += offset;
  679. op(vaddr, len, dir);
  680. kunmap_high(page);
  681. } else if (cache_is_vipt()) {
  682. /* unmapped pages might still be cached */
  683. vaddr = kmap_atomic(page);
  684. op(vaddr + offset, len, dir);
  685. kunmap_atomic(vaddr);
  686. }
  687. } else {
  688. vaddr = page_address(page) + offset;
  689. op(vaddr, len, dir);
  690. }
  691. offset = 0;
  692. page++;
  693. left -= len;
  694. } while (left);
  695. }
  696. /*
  697. * Make an area consistent for devices.
  698. * Note: Drivers should NOT use this function directly, as it will break
  699. * platforms with CONFIG_DMABOUNCE.
  700. * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
  701. */
  702. static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
  703. size_t size, enum dma_data_direction dir)
  704. {
  705. unsigned long paddr;
  706. dma_cache_maint_page(page, off, size, dir, dmac_map_area);
  707. paddr = page_to_phys(page) + off;
  708. if (dir == DMA_FROM_DEVICE) {
  709. outer_inv_range(paddr, paddr + size);
  710. } else {
  711. outer_clean_range(paddr, paddr + size);
  712. }
  713. /* FIXME: non-speculating: flush on bidirectional mappings? */
  714. }
  715. static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
  716. size_t size, enum dma_data_direction dir)
  717. {
  718. unsigned long paddr = page_to_phys(page) + off;
  719. /* FIXME: non-speculating: not required */
  720. /* don't bother invalidating if DMA to device */
  721. if (dir != DMA_TO_DEVICE)
  722. outer_inv_range(paddr, paddr + size);
  723. dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
  724. /*
  725. * Mark the D-cache clean for this page to avoid extra flushing.
  726. */
  727. if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
  728. set_bit(PG_dcache_clean, &page->flags);
  729. }
  730. /**
  731. * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
  732. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  733. * @sg: list of buffers
  734. * @nents: number of buffers to map
  735. * @dir: DMA transfer direction
  736. *
  737. * Map a set of buffers described by scatterlist in streaming mode for DMA.
  738. * This is the scatter-gather version of the dma_map_single interface.
  739. * Here the scatter gather list elements are each tagged with the
  740. * appropriate dma address and length. They are obtained via
  741. * sg_dma_{address,length}.
  742. *
  743. * Device ownership issues as mentioned for dma_map_single are the same
  744. * here.
  745. */
  746. int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  747. enum dma_data_direction dir, struct dma_attrs *attrs)
  748. {
  749. struct dma_map_ops *ops = get_dma_ops(dev);
  750. struct scatterlist *s;
  751. int i, j;
  752. for_each_sg(sg, s, nents, i) {
  753. #ifdef CONFIG_NEED_SG_DMA_LENGTH
  754. s->dma_length = s->length;
  755. #endif
  756. s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
  757. s->length, dir, attrs);
  758. if (dma_mapping_error(dev, s->dma_address))
  759. goto bad_mapping;
  760. }
  761. return nents;
  762. bad_mapping:
  763. for_each_sg(sg, s, i, j)
  764. ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
  765. return 0;
  766. }
  767. /**
  768. * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  769. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  770. * @sg: list of buffers
  771. * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
  772. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  773. *
  774. * Unmap a set of streaming mode DMA translations. Again, CPU access
  775. * rules concerning calls here are the same as for dma_unmap_single().
  776. */
  777. void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
  778. enum dma_data_direction dir, struct dma_attrs *attrs)
  779. {
  780. struct dma_map_ops *ops = get_dma_ops(dev);
  781. struct scatterlist *s;
  782. int i;
  783. for_each_sg(sg, s, nents, i)
  784. ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
  785. }
  786. /**
  787. * arm_dma_sync_sg_for_cpu
  788. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  789. * @sg: list of buffers
  790. * @nents: number of buffers to map (returned from dma_map_sg)
  791. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  792. */
  793. void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  794. int nents, enum dma_data_direction dir)
  795. {
  796. struct dma_map_ops *ops = get_dma_ops(dev);
  797. struct scatterlist *s;
  798. int i;
  799. for_each_sg(sg, s, nents, i)
  800. ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
  801. dir);
  802. }
  803. /**
  804. * arm_dma_sync_sg_for_device
  805. * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  806. * @sg: list of buffers
  807. * @nents: number of buffers to map (returned from dma_map_sg)
  808. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  809. */
  810. void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  811. int nents, enum dma_data_direction dir)
  812. {
  813. struct dma_map_ops *ops = get_dma_ops(dev);
  814. struct scatterlist *s;
  815. int i;
  816. for_each_sg(sg, s, nents, i)
  817. ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
  818. dir);
  819. }
  820. /*
  821. * Return whether the given device DMA address mask can be supported
  822. * properly. For example, if your device can only drive the low 24-bits
  823. * during bus mastering, then you would pass 0x00ffffff as the mask
  824. * to this function.
  825. */
  826. int dma_supported(struct device *dev, u64 mask)
  827. {
  828. if (mask < (u64)arm_dma_limit)
  829. return 0;
  830. return 1;
  831. }
  832. EXPORT_SYMBOL(dma_supported);
  833. static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
  834. {
  835. if (!dev->dma_mask || !dma_supported(dev, dma_mask))
  836. return -EIO;
  837. *dev->dma_mask = dma_mask;
  838. return 0;
  839. }
  840. #define PREALLOC_DMA_DEBUG_ENTRIES 4096
  841. static int __init dma_debug_do_init(void)
  842. {
  843. dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
  844. return 0;
  845. }
  846. fs_initcall(dma_debug_do_init);
  847. #ifdef CONFIG_ARM_DMA_USE_IOMMU
  848. /* IOMMU */
  849. static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
  850. size_t size)
  851. {
  852. unsigned int order = get_order(size);
  853. unsigned int align = 0;
  854. unsigned int count, start;
  855. unsigned long flags;
  856. count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
  857. (1 << mapping->order) - 1) >> mapping->order;
  858. if (order > mapping->order)
  859. align = (1 << (order - mapping->order)) - 1;
  860. spin_lock_irqsave(&mapping->lock, flags);
  861. start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
  862. count, align);
  863. if (start > mapping->bits) {
  864. spin_unlock_irqrestore(&mapping->lock, flags);
  865. return DMA_ERROR_CODE;
  866. }
  867. bitmap_set(mapping->bitmap, start, count);
  868. spin_unlock_irqrestore(&mapping->lock, flags);
  869. return mapping->base + (start << (mapping->order + PAGE_SHIFT));
  870. }
  871. static inline void __free_iova(struct dma_iommu_mapping *mapping,
  872. dma_addr_t addr, size_t size)
  873. {
  874. unsigned int start = (addr - mapping->base) >>
  875. (mapping->order + PAGE_SHIFT);
  876. unsigned int count = ((size >> PAGE_SHIFT) +
  877. (1 << mapping->order) - 1) >> mapping->order;
  878. unsigned long flags;
  879. spin_lock_irqsave(&mapping->lock, flags);
  880. bitmap_clear(mapping->bitmap, start, count);
  881. spin_unlock_irqrestore(&mapping->lock, flags);
  882. }
  883. static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
  884. {
  885. struct page **pages;
  886. int count = size >> PAGE_SHIFT;
  887. int array_size = count * sizeof(struct page *);
  888. int i = 0;
  889. if (array_size <= PAGE_SIZE)
  890. pages = kzalloc(array_size, gfp);
  891. else
  892. pages = vzalloc(array_size);
  893. if (!pages)
  894. return NULL;
  895. while (count) {
  896. int j, order = __fls(count);
  897. pages[i] = alloc_pages(gfp | __GFP_NOWARN, order);
  898. while (!pages[i] && order)
  899. pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order);
  900. if (!pages[i])
  901. goto error;
  902. if (order) {
  903. split_page(pages[i], order);
  904. j = 1 << order;
  905. while (--j)
  906. pages[i + j] = pages[i] + j;
  907. }
  908. __dma_clear_buffer(pages[i], PAGE_SIZE << order);
  909. i += 1 << order;
  910. count -= 1 << order;
  911. }
  912. return pages;
  913. error:
  914. while (i--)
  915. if (pages[i])
  916. __free_pages(pages[i], 0);
  917. if (array_size <= PAGE_SIZE)
  918. kfree(pages);
  919. else
  920. vfree(pages);
  921. return NULL;
  922. }
  923. static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
  924. {
  925. int count = size >> PAGE_SHIFT;
  926. int array_size = count * sizeof(struct page *);
  927. int i;
  928. for (i = 0; i < count; i++)
  929. if (pages[i])
  930. __free_pages(pages[i], 0);
  931. if (array_size <= PAGE_SIZE)
  932. kfree(pages);
  933. else
  934. vfree(pages);
  935. return 0;
  936. }
  937. /*
  938. * Create a CPU mapping for a specified pages
  939. */
  940. static void *
  941. __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
  942. const void *caller)
  943. {
  944. unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
  945. struct vm_struct *area;
  946. unsigned long p;
  947. area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
  948. caller);
  949. if (!area)
  950. return NULL;
  951. area->pages = pages;
  952. area->nr_pages = nr_pages;
  953. p = (unsigned long)area->addr;
  954. for (i = 0; i < nr_pages; i++) {
  955. phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
  956. if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
  957. goto err;
  958. p += PAGE_SIZE;
  959. }
  960. return area->addr;
  961. err:
  962. unmap_kernel_range((unsigned long)area->addr, size);
  963. vunmap(area->addr);
  964. return NULL;
  965. }
  966. /*
  967. * Create a mapping in device IO address space for specified pages
  968. */
  969. static dma_addr_t
  970. __iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
  971. {
  972. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  973. unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  974. dma_addr_t dma_addr, iova;
  975. int i, ret = DMA_ERROR_CODE;
  976. dma_addr = __alloc_iova(mapping, size);
  977. if (dma_addr == DMA_ERROR_CODE)
  978. return dma_addr;
  979. iova = dma_addr;
  980. for (i = 0; i < count; ) {
  981. unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
  982. phys_addr_t phys = page_to_phys(pages[i]);
  983. unsigned int len, j;
  984. for (j = i + 1; j < count; j++, next_pfn++)
  985. if (page_to_pfn(pages[j]) != next_pfn)
  986. break;
  987. len = (j - i) << PAGE_SHIFT;
  988. ret = iommu_map(mapping->domain, iova, phys, len, 0);
  989. if (ret < 0)
  990. goto fail;
  991. iova += len;
  992. i = j;
  993. }
  994. return dma_addr;
  995. fail:
  996. iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
  997. __free_iova(mapping, dma_addr, size);
  998. return DMA_ERROR_CODE;
  999. }
  1000. static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
  1001. {
  1002. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1003. /*
  1004. * add optional in-page offset from iova to size and align
  1005. * result to page size
  1006. */
  1007. size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
  1008. iova &= PAGE_MASK;
  1009. iommu_unmap(mapping->domain, iova, size);
  1010. __free_iova(mapping, iova, size);
  1011. return 0;
  1012. }
  1013. static struct page **__atomic_get_pages(void *addr)
  1014. {
  1015. struct dma_pool *pool = &atomic_pool;
  1016. struct page **pages = pool->pages;
  1017. int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
  1018. return pages + offs;
  1019. }
  1020. static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
  1021. {
  1022. struct vm_struct *area;
  1023. if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
  1024. return __atomic_get_pages(cpu_addr);
  1025. if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
  1026. return cpu_addr;
  1027. area = find_vm_area(cpu_addr);
  1028. if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
  1029. return area->pages;
  1030. return NULL;
  1031. }
  1032. static void *__iommu_alloc_atomic(struct device *dev, size_t size,
  1033. dma_addr_t *handle)
  1034. {
  1035. struct page *page;
  1036. void *addr;
  1037. addr = __alloc_from_pool(size, &page);
  1038. if (!addr)
  1039. return NULL;
  1040. *handle = __iommu_create_mapping(dev, &page, size);
  1041. if (*handle == DMA_ERROR_CODE)
  1042. goto err_mapping;
  1043. return addr;
  1044. err_mapping:
  1045. __free_from_pool(addr, size);
  1046. return NULL;
  1047. }
  1048. static void __iommu_free_atomic(struct device *dev, struct page **pages,
  1049. dma_addr_t handle, size_t size)
  1050. {
  1051. __iommu_remove_mapping(dev, handle, size);
  1052. __free_from_pool(page_address(pages[0]), size);
  1053. }
  1054. static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
  1055. dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
  1056. {
  1057. pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
  1058. struct page **pages;
  1059. void *addr = NULL;
  1060. *handle = DMA_ERROR_CODE;
  1061. size = PAGE_ALIGN(size);
  1062. if (gfp & GFP_ATOMIC)
  1063. return __iommu_alloc_atomic(dev, size, handle);
  1064. pages = __iommu_alloc_buffer(dev, size, gfp);
  1065. if (!pages)
  1066. return NULL;
  1067. *handle = __iommu_create_mapping(dev, pages, size);
  1068. if (*handle == DMA_ERROR_CODE)
  1069. goto err_buffer;
  1070. if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
  1071. return pages;
  1072. addr = __iommu_alloc_remap(pages, size, gfp, prot,
  1073. __builtin_return_address(0));
  1074. if (!addr)
  1075. goto err_mapping;
  1076. return addr;
  1077. err_mapping:
  1078. __iommu_remove_mapping(dev, *handle, size);
  1079. err_buffer:
  1080. __iommu_free_buffer(dev, pages, size);
  1081. return NULL;
  1082. }
  1083. static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
  1084. void *cpu_addr, dma_addr_t dma_addr, size_t size,
  1085. struct dma_attrs *attrs)
  1086. {
  1087. unsigned long uaddr = vma->vm_start;
  1088. unsigned long usize = vma->vm_end - vma->vm_start;
  1089. struct page **pages = __iommu_get_pages(cpu_addr, attrs);
  1090. vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
  1091. if (!pages)
  1092. return -ENXIO;
  1093. do {
  1094. int ret = vm_insert_page(vma, uaddr, *pages++);
  1095. if (ret) {
  1096. pr_err("Remapping memory failed: %d\n", ret);
  1097. return ret;
  1098. }
  1099. uaddr += PAGE_SIZE;
  1100. usize -= PAGE_SIZE;
  1101. } while (usize > 0);
  1102. return 0;
  1103. }
  1104. /*
  1105. * free a page as defined by the above mapping.
  1106. * Must not be called with IRQs disabled.
  1107. */
  1108. void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
  1109. dma_addr_t handle, struct dma_attrs *attrs)
  1110. {
  1111. struct page **pages = __iommu_get_pages(cpu_addr, attrs);
  1112. size = PAGE_ALIGN(size);
  1113. if (!pages) {
  1114. WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
  1115. return;
  1116. }
  1117. if (__in_atomic_pool(cpu_addr, size)) {
  1118. __iommu_free_atomic(dev, pages, handle, size);
  1119. return;
  1120. }
  1121. if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
  1122. unmap_kernel_range((unsigned long)cpu_addr, size);
  1123. vunmap(cpu_addr);
  1124. }
  1125. __iommu_remove_mapping(dev, handle, size);
  1126. __iommu_free_buffer(dev, pages, size);
  1127. }
  1128. static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
  1129. void *cpu_addr, dma_addr_t dma_addr,
  1130. size_t size, struct dma_attrs *attrs)
  1131. {
  1132. unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  1133. struct page **pages = __iommu_get_pages(cpu_addr, attrs);
  1134. if (!pages)
  1135. return -ENXIO;
  1136. return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
  1137. GFP_KERNEL);
  1138. }
  1139. /*
  1140. * Map a part of the scatter-gather list into contiguous io address space
  1141. */
  1142. static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
  1143. size_t size, dma_addr_t *handle,
  1144. enum dma_data_direction dir, struct dma_attrs *attrs,
  1145. bool is_coherent)
  1146. {
  1147. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1148. dma_addr_t iova, iova_base;
  1149. int ret = 0;
  1150. unsigned int count;
  1151. struct scatterlist *s;
  1152. size = PAGE_ALIGN(size);
  1153. *handle = DMA_ERROR_CODE;
  1154. iova_base = iova = __alloc_iova(mapping, size);
  1155. if (iova == DMA_ERROR_CODE)
  1156. return -ENOMEM;
  1157. for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
  1158. phys_addr_t phys = page_to_phys(sg_page(s));
  1159. unsigned int len = PAGE_ALIGN(s->offset + s->length);
  1160. if (!is_coherent &&
  1161. !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  1162. __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
  1163. ret = iommu_map(mapping->domain, iova, phys, len, 0);
  1164. if (ret < 0)
  1165. goto fail;
  1166. count += len >> PAGE_SHIFT;
  1167. iova += len;
  1168. }
  1169. *handle = iova_base;
  1170. return 0;
  1171. fail:
  1172. iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
  1173. __free_iova(mapping, iova_base, size);
  1174. return ret;
  1175. }
  1176. static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  1177. enum dma_data_direction dir, struct dma_attrs *attrs,
  1178. bool is_coherent)
  1179. {
  1180. struct scatterlist *s = sg, *dma = sg, *start = sg;
  1181. int i, count = 0;
  1182. unsigned int offset = s->offset;
  1183. unsigned int size = s->offset + s->length;
  1184. unsigned int max = dma_get_max_seg_size(dev);
  1185. for (i = 1; i < nents; i++) {
  1186. s = sg_next(s);
  1187. s->dma_address = DMA_ERROR_CODE;
  1188. s->dma_length = 0;
  1189. if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
  1190. if (__map_sg_chunk(dev, start, size, &dma->dma_address,
  1191. dir, attrs, is_coherent) < 0)
  1192. goto bad_mapping;
  1193. dma->dma_address += offset;
  1194. dma->dma_length = size - offset;
  1195. size = offset = s->offset;
  1196. start = s;
  1197. dma = sg_next(dma);
  1198. count += 1;
  1199. }
  1200. size += s->length;
  1201. }
  1202. if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
  1203. is_coherent) < 0)
  1204. goto bad_mapping;
  1205. dma->dma_address += offset;
  1206. dma->dma_length = size - offset;
  1207. return count+1;
  1208. bad_mapping:
  1209. for_each_sg(sg, s, count, i)
  1210. __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
  1211. return 0;
  1212. }
  1213. /**
  1214. * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
  1215. * @dev: valid struct device pointer
  1216. * @sg: list of buffers
  1217. * @nents: number of buffers to map
  1218. * @dir: DMA transfer direction
  1219. *
  1220. * Map a set of i/o coherent buffers described by scatterlist in streaming
  1221. * mode for DMA. The scatter gather list elements are merged together (if
  1222. * possible) and tagged with the appropriate dma address and length. They are
  1223. * obtained via sg_dma_{address,length}.
  1224. */
  1225. int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
  1226. int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
  1227. {
  1228. return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
  1229. }
  1230. /**
  1231. * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
  1232. * @dev: valid struct device pointer
  1233. * @sg: list of buffers
  1234. * @nents: number of buffers to map
  1235. * @dir: DMA transfer direction
  1236. *
  1237. * Map a set of buffers described by scatterlist in streaming mode for DMA.
  1238. * The scatter gather list elements are merged together (if possible) and
  1239. * tagged with the appropriate dma address and length. They are obtained via
  1240. * sg_dma_{address,length}.
  1241. */
  1242. int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
  1243. int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
  1244. {
  1245. return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
  1246. }
  1247. static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
  1248. int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
  1249. bool is_coherent)
  1250. {
  1251. struct scatterlist *s;
  1252. int i;
  1253. for_each_sg(sg, s, nents, i) {
  1254. if (sg_dma_len(s))
  1255. __iommu_remove_mapping(dev, sg_dma_address(s),
  1256. sg_dma_len(s));
  1257. if (!is_coherent &&
  1258. !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  1259. __dma_page_dev_to_cpu(sg_page(s), s->offset,
  1260. s->length, dir);
  1261. }
  1262. }
  1263. /**
  1264. * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  1265. * @dev: valid struct device pointer
  1266. * @sg: list of buffers
  1267. * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
  1268. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1269. *
  1270. * Unmap a set of streaming mode DMA translations. Again, CPU access
  1271. * rules concerning calls here are the same as for dma_unmap_single().
  1272. */
  1273. void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
  1274. int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
  1275. {
  1276. __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
  1277. }
  1278. /**
  1279. * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  1280. * @dev: valid struct device pointer
  1281. * @sg: list of buffers
  1282. * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
  1283. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1284. *
  1285. * Unmap a set of streaming mode DMA translations. Again, CPU access
  1286. * rules concerning calls here are the same as for dma_unmap_single().
  1287. */
  1288. void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
  1289. enum dma_data_direction dir, struct dma_attrs *attrs)
  1290. {
  1291. __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
  1292. }
  1293. /**
  1294. * arm_iommu_sync_sg_for_cpu
  1295. * @dev: valid struct device pointer
  1296. * @sg: list of buffers
  1297. * @nents: number of buffers to map (returned from dma_map_sg)
  1298. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1299. */
  1300. void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  1301. int nents, enum dma_data_direction dir)
  1302. {
  1303. struct scatterlist *s;
  1304. int i;
  1305. for_each_sg(sg, s, nents, i)
  1306. __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
  1307. }
  1308. /**
  1309. * arm_iommu_sync_sg_for_device
  1310. * @dev: valid struct device pointer
  1311. * @sg: list of buffers
  1312. * @nents: number of buffers to map (returned from dma_map_sg)
  1313. * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  1314. */
  1315. void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  1316. int nents, enum dma_data_direction dir)
  1317. {
  1318. struct scatterlist *s;
  1319. int i;
  1320. for_each_sg(sg, s, nents, i)
  1321. __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
  1322. }
  1323. /**
  1324. * arm_coherent_iommu_map_page
  1325. * @dev: valid struct device pointer
  1326. * @page: page that buffer resides in
  1327. * @offset: offset into page for start of buffer
  1328. * @size: size of buffer to map
  1329. * @dir: DMA transfer direction
  1330. *
  1331. * Coherent IOMMU aware version of arm_dma_map_page()
  1332. */
  1333. static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
  1334. unsigned long offset, size_t size, enum dma_data_direction dir,
  1335. struct dma_attrs *attrs)
  1336. {
  1337. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1338. dma_addr_t dma_addr;
  1339. int ret, len = PAGE_ALIGN(size + offset);
  1340. dma_addr = __alloc_iova(mapping, len);
  1341. if (dma_addr == DMA_ERROR_CODE)
  1342. return dma_addr;
  1343. ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
  1344. if (ret < 0)
  1345. goto fail;
  1346. return dma_addr + offset;
  1347. fail:
  1348. __free_iova(mapping, dma_addr, len);
  1349. return DMA_ERROR_CODE;
  1350. }
  1351. /**
  1352. * arm_iommu_map_page
  1353. * @dev: valid struct device pointer
  1354. * @page: page that buffer resides in
  1355. * @offset: offset into page for start of buffer
  1356. * @size: size of buffer to map
  1357. * @dir: DMA transfer direction
  1358. *
  1359. * IOMMU aware version of arm_dma_map_page()
  1360. */
  1361. static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
  1362. unsigned long offset, size_t size, enum dma_data_direction dir,
  1363. struct dma_attrs *attrs)
  1364. {
  1365. if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  1366. __dma_page_cpu_to_dev(page, offset, size, dir);
  1367. return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
  1368. }
  1369. /**
  1370. * arm_coherent_iommu_unmap_page
  1371. * @dev: valid struct device pointer
  1372. * @handle: DMA address of buffer
  1373. * @size: size of buffer (same as passed to dma_map_page)
  1374. * @dir: DMA transfer direction (same as passed to dma_map_page)
  1375. *
  1376. * Coherent IOMMU aware version of arm_dma_unmap_page()
  1377. */
  1378. static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
  1379. size_t size, enum dma_data_direction dir,
  1380. struct dma_attrs *attrs)
  1381. {
  1382. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1383. dma_addr_t iova = handle & PAGE_MASK;
  1384. int offset = handle & ~PAGE_MASK;
  1385. int len = PAGE_ALIGN(size + offset);
  1386. if (!iova)
  1387. return;
  1388. iommu_unmap(mapping->domain, iova, len);
  1389. __free_iova(mapping, iova, len);
  1390. }
  1391. /**
  1392. * arm_iommu_unmap_page
  1393. * @dev: valid struct device pointer
  1394. * @handle: DMA address of buffer
  1395. * @size: size of buffer (same as passed to dma_map_page)
  1396. * @dir: DMA transfer direction (same as passed to dma_map_page)
  1397. *
  1398. * IOMMU aware version of arm_dma_unmap_page()
  1399. */
  1400. static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
  1401. size_t size, enum dma_data_direction dir,
  1402. struct dma_attrs *attrs)
  1403. {
  1404. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1405. dma_addr_t iova = handle & PAGE_MASK;
  1406. struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1407. int offset = handle & ~PAGE_MASK;
  1408. int len = PAGE_ALIGN(size + offset);
  1409. if (!iova)
  1410. return;
  1411. if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
  1412. __dma_page_dev_to_cpu(page, offset, size, dir);
  1413. iommu_unmap(mapping->domain, iova, len);
  1414. __free_iova(mapping, iova, len);
  1415. }
  1416. static void arm_iommu_sync_single_for_cpu(struct device *dev,
  1417. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  1418. {
  1419. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1420. dma_addr_t iova = handle & PAGE_MASK;
  1421. struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1422. unsigned int offset = handle & ~PAGE_MASK;
  1423. if (!iova)
  1424. return;
  1425. __dma_page_dev_to_cpu(page, offset, size, dir);
  1426. }
  1427. static void arm_iommu_sync_single_for_device(struct device *dev,
  1428. dma_addr_t handle, size_t size, enum dma_data_direction dir)
  1429. {
  1430. struct dma_iommu_mapping *mapping = dev->archdata.mapping;
  1431. dma_addr_t iova = handle & PAGE_MASK;
  1432. struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
  1433. unsigned int offset = handle & ~PAGE_MASK;
  1434. if (!iova)
  1435. return;
  1436. __dma_page_cpu_to_dev(page, offset, size, dir);
  1437. }
  1438. struct dma_map_ops iommu_ops = {
  1439. .alloc = arm_iommu_alloc_attrs,
  1440. .free = arm_iommu_free_attrs,
  1441. .mmap = arm_iommu_mmap_attrs,
  1442. .get_sgtable = arm_iommu_get_sgtable,
  1443. .map_page = arm_iommu_map_page,
  1444. .unmap_page = arm_iommu_unmap_page,
  1445. .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
  1446. .sync_single_for_device = arm_iommu_sync_single_for_device,
  1447. .map_sg = arm_iommu_map_sg,
  1448. .unmap_sg = arm_iommu_unmap_sg,
  1449. .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
  1450. .sync_sg_for_device = arm_iommu_sync_sg_for_device,
  1451. };
  1452. struct dma_map_ops iommu_coherent_ops = {
  1453. .alloc = arm_iommu_alloc_attrs,
  1454. .free = arm_iommu_free_attrs,
  1455. .mmap = arm_iommu_mmap_attrs,
  1456. .get_sgtable = arm_iommu_get_sgtable,
  1457. .map_page = arm_coherent_iommu_map_page,
  1458. .unmap_page = arm_coherent_iommu_unmap_page,
  1459. .map_sg = arm_coherent_iommu_map_sg,
  1460. .unmap_sg = arm_coherent_iommu_unmap_sg,
  1461. };
  1462. /**
  1463. * arm_iommu_create_mapping
  1464. * @bus: pointer to the bus holding the client device (for IOMMU calls)
  1465. * @base: start address of the valid IO address space
  1466. * @size: size of the valid IO address space
  1467. * @order: accuracy of the IO addresses allocations
  1468. *
  1469. * Creates a mapping structure which holds information about used/unused
  1470. * IO address ranges, which is required to perform memory allocation and
  1471. * mapping with IOMMU aware functions.
  1472. *
  1473. * The client device need to be attached to the mapping with
  1474. * arm_iommu_attach_device function.
  1475. */
  1476. struct dma_iommu_mapping *
  1477. arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
  1478. int order)
  1479. {
  1480. unsigned int count = size >> (PAGE_SHIFT + order);
  1481. unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
  1482. struct dma_iommu_mapping *mapping;
  1483. int err = -ENOMEM;
  1484. if (!count)
  1485. return ERR_PTR(-EINVAL);
  1486. mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
  1487. if (!mapping)
  1488. goto err;
  1489. mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  1490. if (!mapping->bitmap)
  1491. goto err2;
  1492. mapping->base = base;
  1493. mapping->bits = BITS_PER_BYTE * bitmap_size;
  1494. mapping->order = order;
  1495. spin_lock_init(&mapping->lock);
  1496. mapping->domain = iommu_domain_alloc(bus);
  1497. if (!mapping->domain)
  1498. goto err3;
  1499. kref_init(&mapping->kref);
  1500. return mapping;
  1501. err3:
  1502. kfree(mapping->bitmap);
  1503. err2:
  1504. kfree(mapping);
  1505. err:
  1506. return ERR_PTR(err);
  1507. }
  1508. static void release_iommu_mapping(struct kref *kref)
  1509. {
  1510. struct dma_iommu_mapping *mapping =
  1511. container_of(kref, struct dma_iommu_mapping, kref);
  1512. iommu_domain_free(mapping->domain);
  1513. kfree(mapping->bitmap);
  1514. kfree(mapping);
  1515. }
  1516. void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
  1517. {
  1518. if (mapping)
  1519. kref_put(&mapping->kref, release_iommu_mapping);
  1520. }
  1521. /**
  1522. * arm_iommu_attach_device
  1523. * @dev: valid struct device pointer
  1524. * @mapping: io address space mapping structure (returned from
  1525. * arm_iommu_create_mapping)
  1526. *
  1527. * Attaches specified io address space mapping to the provided device,
  1528. * this replaces the dma operations (dma_map_ops pointer) with the
  1529. * IOMMU aware version. More than one client might be attached to
  1530. * the same io address space mapping.
  1531. */
  1532. int arm_iommu_attach_device(struct device *dev,
  1533. struct dma_iommu_mapping *mapping)
  1534. {
  1535. int err;
  1536. err = iommu_attach_device(mapping->domain, dev);
  1537. if (err)
  1538. return err;
  1539. kref_get(&mapping->kref);
  1540. dev->archdata.mapping = mapping;
  1541. set_dma_ops(dev, &iommu_ops);
  1542. pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
  1543. return 0;
  1544. }
  1545. #endif