dma-mapping.h 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. #ifndef _ASM_X86_DMA_MAPPING_H
  2. #define _ASM_X86_DMA_MAPPING_H
  3. /*
  4. * IOMMU interface. See Documentation/PCI/PCI-DMA-mapping.txt and
  5. * Documentation/DMA-API.txt for documentation.
  6. */
  7. #include <linux/kmemcheck.h>
  8. #include <linux/scatterlist.h>
  9. #include <linux/dma-debug.h>
  10. #include <linux/dma-attrs.h>
  11. #include <asm/io.h>
  12. #include <asm/swiotlb.h>
  13. #include <asm-generic/dma-coherent.h>
  14. extern dma_addr_t bad_dma_address;
  15. extern int iommu_merge;
  16. extern struct device x86_dma_fallback_dev;
  17. extern int panic_on_overflow;
  18. extern struct dma_map_ops *dma_ops;
  19. static inline struct dma_map_ops *get_dma_ops(struct device *dev)
  20. {
  21. #ifdef CONFIG_X86_32
  22. return dma_ops;
  23. #else
  24. if (unlikely(!dev) || !dev->archdata.dma_ops)
  25. return dma_ops;
  26. else
  27. return dev->archdata.dma_ops;
  28. #endif
  29. }
  30. /* Make sure we keep the same behaviour */
  31. static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
  32. {
  33. struct dma_map_ops *ops = get_dma_ops(dev);
  34. if (ops->mapping_error)
  35. return ops->mapping_error(dev, dma_addr);
  36. return (dma_addr == bad_dma_address);
  37. }
  38. #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
  39. #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
  40. #define dma_is_consistent(d, h) (1)
  41. extern int dma_supported(struct device *hwdev, u64 mask);
  42. extern int dma_set_mask(struct device *dev, u64 mask);
  43. extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
  44. dma_addr_t *dma_addr, gfp_t flag);
  45. static inline dma_addr_t
  46. dma_map_single(struct device *hwdev, void *ptr, size_t size,
  47. enum dma_data_direction dir)
  48. {
  49. struct dma_map_ops *ops = get_dma_ops(hwdev);
  50. dma_addr_t addr;
  51. BUG_ON(!valid_dma_direction(dir));
  52. kmemcheck_mark_initialized(ptr, size);
  53. addr = ops->map_page(hwdev, virt_to_page(ptr),
  54. (unsigned long)ptr & ~PAGE_MASK, size,
  55. dir, NULL);
  56. debug_dma_map_page(hwdev, virt_to_page(ptr),
  57. (unsigned long)ptr & ~PAGE_MASK, size,
  58. dir, addr, true);
  59. return addr;
  60. }
  61. static inline void
  62. dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
  63. enum dma_data_direction dir)
  64. {
  65. struct dma_map_ops *ops = get_dma_ops(dev);
  66. BUG_ON(!valid_dma_direction(dir));
  67. if (ops->unmap_page)
  68. ops->unmap_page(dev, addr, size, dir, NULL);
  69. debug_dma_unmap_page(dev, addr, size, dir, true);
  70. }
  71. static inline int
  72. dma_map_sg(struct device *hwdev, struct scatterlist *sg,
  73. int nents, enum dma_data_direction dir)
  74. {
  75. struct dma_map_ops *ops = get_dma_ops(hwdev);
  76. int ents;
  77. BUG_ON(!valid_dma_direction(dir));
  78. ents = ops->map_sg(hwdev, sg, nents, dir, NULL);
  79. debug_dma_map_sg(hwdev, sg, nents, ents, dir);
  80. return ents;
  81. }
  82. static inline void
  83. dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
  84. enum dma_data_direction dir)
  85. {
  86. struct dma_map_ops *ops = get_dma_ops(hwdev);
  87. BUG_ON(!valid_dma_direction(dir));
  88. debug_dma_unmap_sg(hwdev, sg, nents, dir);
  89. if (ops->unmap_sg)
  90. ops->unmap_sg(hwdev, sg, nents, dir, NULL);
  91. }
  92. static inline void
  93. dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
  94. size_t size, enum dma_data_direction dir)
  95. {
  96. struct dma_map_ops *ops = get_dma_ops(hwdev);
  97. BUG_ON(!valid_dma_direction(dir));
  98. if (ops->sync_single_for_cpu)
  99. ops->sync_single_for_cpu(hwdev, dma_handle, size, dir);
  100. debug_dma_sync_single_for_cpu(hwdev, dma_handle, size, dir);
  101. flush_write_buffers();
  102. }
  103. static inline void
  104. dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle,
  105. size_t size, enum dma_data_direction dir)
  106. {
  107. struct dma_map_ops *ops = get_dma_ops(hwdev);
  108. BUG_ON(!valid_dma_direction(dir));
  109. if (ops->sync_single_for_device)
  110. ops->sync_single_for_device(hwdev, dma_handle, size, dir);
  111. debug_dma_sync_single_for_device(hwdev, dma_handle, size, dir);
  112. flush_write_buffers();
  113. }
  114. static inline void
  115. dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
  116. unsigned long offset, size_t size,
  117. enum dma_data_direction dir)
  118. {
  119. struct dma_map_ops *ops = get_dma_ops(hwdev);
  120. BUG_ON(!valid_dma_direction(dir));
  121. if (ops->sync_single_range_for_cpu)
  122. ops->sync_single_range_for_cpu(hwdev, dma_handle, offset,
  123. size, dir);
  124. debug_dma_sync_single_range_for_cpu(hwdev, dma_handle,
  125. offset, size, dir);
  126. flush_write_buffers();
  127. }
  128. static inline void
  129. dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle,
  130. unsigned long offset, size_t size,
  131. enum dma_data_direction dir)
  132. {
  133. struct dma_map_ops *ops = get_dma_ops(hwdev);
  134. BUG_ON(!valid_dma_direction(dir));
  135. if (ops->sync_single_range_for_device)
  136. ops->sync_single_range_for_device(hwdev, dma_handle,
  137. offset, size, dir);
  138. debug_dma_sync_single_range_for_device(hwdev, dma_handle,
  139. offset, size, dir);
  140. flush_write_buffers();
  141. }
  142. static inline void
  143. dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
  144. int nelems, enum dma_data_direction dir)
  145. {
  146. struct dma_map_ops *ops = get_dma_ops(hwdev);
  147. BUG_ON(!valid_dma_direction(dir));
  148. if (ops->sync_sg_for_cpu)
  149. ops->sync_sg_for_cpu(hwdev, sg, nelems, dir);
  150. debug_dma_sync_sg_for_cpu(hwdev, sg, nelems, dir);
  151. flush_write_buffers();
  152. }
  153. static inline void
  154. dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
  155. int nelems, enum dma_data_direction dir)
  156. {
  157. struct dma_map_ops *ops = get_dma_ops(hwdev);
  158. BUG_ON(!valid_dma_direction(dir));
  159. if (ops->sync_sg_for_device)
  160. ops->sync_sg_for_device(hwdev, sg, nelems, dir);
  161. debug_dma_sync_sg_for_device(hwdev, sg, nelems, dir);
  162. flush_write_buffers();
  163. }
  164. static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
  165. size_t offset, size_t size,
  166. enum dma_data_direction dir)
  167. {
  168. struct dma_map_ops *ops = get_dma_ops(dev);
  169. dma_addr_t addr;
  170. BUG_ON(!valid_dma_direction(dir));
  171. addr = ops->map_page(dev, page, offset, size, dir, NULL);
  172. debug_dma_map_page(dev, page, offset, size, dir, addr, false);
  173. return addr;
  174. }
  175. static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
  176. size_t size, enum dma_data_direction dir)
  177. {
  178. struct dma_map_ops *ops = get_dma_ops(dev);
  179. BUG_ON(!valid_dma_direction(dir));
  180. if (ops->unmap_page)
  181. ops->unmap_page(dev, addr, size, dir, NULL);
  182. debug_dma_unmap_page(dev, addr, size, dir, false);
  183. }
  184. static inline void
  185. dma_cache_sync(struct device *dev, void *vaddr, size_t size,
  186. enum dma_data_direction dir)
  187. {
  188. flush_write_buffers();
  189. }
  190. static inline int dma_get_cache_alignment(void)
  191. {
  192. /* no easy way to get cache size on all x86, so return the
  193. * maximum possible, to be safe */
  194. return boot_cpu_data.x86_clflush_size;
  195. }
  196. static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
  197. gfp_t gfp)
  198. {
  199. unsigned long dma_mask = 0;
  200. dma_mask = dev->coherent_dma_mask;
  201. if (!dma_mask)
  202. dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32);
  203. return dma_mask;
  204. }
  205. static inline gfp_t dma_alloc_coherent_gfp_flags(struct device *dev, gfp_t gfp)
  206. {
  207. unsigned long dma_mask = dma_alloc_coherent_mask(dev, gfp);
  208. if (dma_mask <= DMA_BIT_MASK(24))
  209. gfp |= GFP_DMA;
  210. #ifdef CONFIG_X86_64
  211. if (dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
  212. gfp |= GFP_DMA32;
  213. #endif
  214. return gfp;
  215. }
  216. static inline void *
  217. dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
  218. gfp_t gfp)
  219. {
  220. struct dma_map_ops *ops = get_dma_ops(dev);
  221. void *memory;
  222. gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
  223. if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
  224. return memory;
  225. if (!dev) {
  226. dev = &x86_dma_fallback_dev;
  227. gfp |= GFP_DMA;
  228. }
  229. if (!is_device_dma_capable(dev))
  230. return NULL;
  231. if (!ops->alloc_coherent)
  232. return NULL;
  233. memory = ops->alloc_coherent(dev, size, dma_handle,
  234. dma_alloc_coherent_gfp_flags(dev, gfp));
  235. debug_dma_alloc_coherent(dev, size, *dma_handle, memory);
  236. return memory;
  237. }
  238. static inline void dma_free_coherent(struct device *dev, size_t size,
  239. void *vaddr, dma_addr_t bus)
  240. {
  241. struct dma_map_ops *ops = get_dma_ops(dev);
  242. WARN_ON(irqs_disabled()); /* for portability */
  243. if (dma_release_from_coherent(dev, get_order(size), vaddr))
  244. return;
  245. debug_dma_free_coherent(dev, size, vaddr, bus);
  246. if (ops->free_coherent)
  247. ops->free_coherent(dev, size, vaddr, bus);
  248. }
  249. #endif