flush.c 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. /*
  2. * linux/arch/arm/mm/flush.c
  3. *
  4. * Copyright (C) 1995-2002 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/mm.h>
  12. #include <linux/pagemap.h>
  13. #include <linux/highmem.h>
  14. #include <asm/cacheflush.h>
  15. #include <asm/cachetype.h>
  16. #include <asm/highmem.h>
  17. #include <asm/smp_plat.h>
  18. #include <asm/tlbflush.h>
  19. #include "mm.h"
  20. #ifdef CONFIG_CPU_CACHE_VIPT
  21. static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  22. {
  23. unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  24. const int zero = 0;
  25. set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
  26. asm( "mcrr p15, 0, %1, %0, c14\n"
  27. " mcr p15, 0, %2, c7, c10, 4"
  28. :
  29. : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero)
  30. : "cc");
  31. }
  32. static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
  33. {
  34. unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  35. unsigned long offset = vaddr & (PAGE_SIZE - 1);
  36. unsigned long to;
  37. set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
  38. to = va + offset;
  39. flush_icache_range(to, to + len);
  40. }
  41. void flush_cache_mm(struct mm_struct *mm)
  42. {
  43. if (cache_is_vivt()) {
  44. vivt_flush_cache_mm(mm);
  45. return;
  46. }
  47. if (cache_is_vipt_aliasing()) {
  48. asm( "mcr p15, 0, %0, c7, c14, 0\n"
  49. " mcr p15, 0, %0, c7, c10, 4"
  50. :
  51. : "r" (0)
  52. : "cc");
  53. }
  54. }
  55. void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  56. {
  57. if (cache_is_vivt()) {
  58. vivt_flush_cache_range(vma, start, end);
  59. return;
  60. }
  61. if (cache_is_vipt_aliasing()) {
  62. asm( "mcr p15, 0, %0, c7, c14, 0\n"
  63. " mcr p15, 0, %0, c7, c10, 4"
  64. :
  65. : "r" (0)
  66. : "cc");
  67. }
  68. if (vma->vm_flags & VM_EXEC)
  69. __flush_icache_all();
  70. }
  71. void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  72. {
  73. if (cache_is_vivt()) {
  74. vivt_flush_cache_page(vma, user_addr, pfn);
  75. return;
  76. }
  77. if (cache_is_vipt_aliasing()) {
  78. flush_pfn_alias(pfn, user_addr);
  79. __flush_icache_all();
  80. }
  81. if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
  82. __flush_icache_all();
  83. }
  84. #else
  85. #define flush_pfn_alias(pfn,vaddr) do { } while (0)
  86. #define flush_icache_alias(pfn,vaddr,len) do { } while (0)
  87. #endif
  88. static void flush_ptrace_access_other(void *args)
  89. {
  90. __flush_icache_all();
  91. }
  92. static
  93. void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  94. unsigned long uaddr, void *kaddr, unsigned long len)
  95. {
  96. if (cache_is_vivt()) {
  97. if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
  98. unsigned long addr = (unsigned long)kaddr;
  99. __cpuc_coherent_kern_range(addr, addr + len);
  100. }
  101. return;
  102. }
  103. if (cache_is_vipt_aliasing()) {
  104. flush_pfn_alias(page_to_pfn(page), uaddr);
  105. __flush_icache_all();
  106. return;
  107. }
  108. /* VIPT non-aliasing D-cache */
  109. if (vma->vm_flags & VM_EXEC) {
  110. unsigned long addr = (unsigned long)kaddr;
  111. if (icache_is_vipt_aliasing())
  112. flush_icache_alias(page_to_pfn(page), uaddr, len);
  113. else
  114. __cpuc_coherent_kern_range(addr, addr + len);
  115. if (cache_ops_need_broadcast())
  116. smp_call_function(flush_ptrace_access_other,
  117. NULL, 1);
  118. }
  119. }
  120. /*
  121. * Copy user data from/to a page which is mapped into a different
  122. * processes address space. Really, we want to allow our "user
  123. * space" model to handle this.
  124. *
  125. * Note that this code needs to run on the current CPU.
  126. */
  127. void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
  128. unsigned long uaddr, void *dst, const void *src,
  129. unsigned long len)
  130. {
  131. #ifdef CONFIG_SMP
  132. preempt_disable();
  133. #endif
  134. memcpy(dst, src, len);
  135. flush_ptrace_access(vma, page, uaddr, dst, len);
  136. #ifdef CONFIG_SMP
  137. preempt_enable();
  138. #endif
  139. }
  140. void __flush_dcache_page(struct address_space *mapping, struct page *page)
  141. {
  142. /*
  143. * Writeback any data associated with the kernel mapping of this
  144. * page. This ensures that data in the physical page is mutually
  145. * coherent with the kernels mapping.
  146. */
  147. if (!PageHighMem(page)) {
  148. __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
  149. } else {
  150. void *addr;
  151. if (cache_is_vipt_nonaliasing()) {
  152. addr = kmap_atomic(page);
  153. __cpuc_flush_dcache_area(addr, PAGE_SIZE);
  154. kunmap_atomic(addr);
  155. } else {
  156. addr = kmap_high_get(page);
  157. if (addr) {
  158. __cpuc_flush_dcache_area(addr, PAGE_SIZE);
  159. kunmap_high(page);
  160. }
  161. }
  162. }
  163. /*
  164. * If this is a page cache page, and we have an aliasing VIPT cache,
  165. * we only need to do one flush - which would be at the relevant
  166. * userspace colour, which is congruent with page->index.
  167. */
  168. if (mapping && cache_is_vipt_aliasing())
  169. flush_pfn_alias(page_to_pfn(page),
  170. page->index << PAGE_CACHE_SHIFT);
  171. }
  172. static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
  173. {
  174. struct mm_struct *mm = current->active_mm;
  175. struct vm_area_struct *mpnt;
  176. pgoff_t pgoff;
  177. /*
  178. * There are possible user space mappings of this page:
  179. * - VIVT cache: we need to also write back and invalidate all user
  180. * data in the current VM view associated with this page.
  181. * - aliasing VIPT: we only need to find one mapping of this page.
  182. */
  183. pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  184. flush_dcache_mmap_lock(mapping);
  185. vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
  186. unsigned long offset;
  187. /*
  188. * If this VMA is not in our MM, we can ignore it.
  189. */
  190. if (mpnt->vm_mm != mm)
  191. continue;
  192. if (!(mpnt->vm_flags & VM_MAYSHARE))
  193. continue;
  194. offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
  195. flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
  196. }
  197. flush_dcache_mmap_unlock(mapping);
  198. }
  199. #if __LINUX_ARM_ARCH__ >= 6
  200. void __sync_icache_dcache(pte_t pteval)
  201. {
  202. unsigned long pfn;
  203. struct page *page;
  204. struct address_space *mapping;
  205. if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
  206. /* only flush non-aliasing VIPT caches for exec mappings */
  207. return;
  208. pfn = pte_pfn(pteval);
  209. if (!pfn_valid(pfn))
  210. return;
  211. page = pfn_to_page(pfn);
  212. if (cache_is_vipt_aliasing())
  213. mapping = page_mapping(page);
  214. else
  215. mapping = NULL;
  216. if (!test_and_set_bit(PG_dcache_clean, &page->flags))
  217. __flush_dcache_page(mapping, page);
  218. if (pte_exec(pteval))
  219. __flush_icache_all();
  220. }
  221. #endif
  222. /*
  223. * Ensure cache coherency between kernel mapping and userspace mapping
  224. * of this page.
  225. *
  226. * We have three cases to consider:
  227. * - VIPT non-aliasing cache: fully coherent so nothing required.
  228. * - VIVT: fully aliasing, so we need to handle every alias in our
  229. * current VM view.
  230. * - VIPT aliasing: need to handle one alias in our current VM view.
  231. *
  232. * If we need to handle aliasing:
  233. * If the page only exists in the page cache and there are no user
  234. * space mappings, we can be lazy and remember that we may have dirty
  235. * kernel cache lines for later. Otherwise, we assume we have
  236. * aliasing mappings.
  237. *
  238. * Note that we disable the lazy flush for SMP configurations where
  239. * the cache maintenance operations are not automatically broadcasted.
  240. */
  241. void flush_dcache_page(struct page *page)
  242. {
  243. struct address_space *mapping;
  244. /*
  245. * The zero page is never written to, so never has any dirty
  246. * cache lines, and therefore never needs to be flushed.
  247. */
  248. if (page == ZERO_PAGE(0))
  249. return;
  250. mapping = page_mapping(page);
  251. if (!cache_ops_need_broadcast() &&
  252. mapping && !mapping_mapped(mapping))
  253. clear_bit(PG_dcache_clean, &page->flags);
  254. else {
  255. __flush_dcache_page(mapping, page);
  256. if (mapping && cache_is_vivt())
  257. __flush_dcache_aliases(mapping, page);
  258. else if (mapping)
  259. __flush_icache_all();
  260. set_bit(PG_dcache_clean, &page->flags);
  261. }
  262. }
  263. EXPORT_SYMBOL(flush_dcache_page);
  264. /*
  265. * Flush an anonymous page so that users of get_user_pages()
  266. * can safely access the data. The expected sequence is:
  267. *
  268. * get_user_pages()
  269. * -> flush_anon_page
  270. * memcpy() to/from page
  271. * if written to page, flush_dcache_page()
  272. */
  273. void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
  274. {
  275. unsigned long pfn;
  276. /* VIPT non-aliasing caches need do nothing */
  277. if (cache_is_vipt_nonaliasing())
  278. return;
  279. /*
  280. * Write back and invalidate userspace mapping.
  281. */
  282. pfn = page_to_pfn(page);
  283. if (cache_is_vivt()) {
  284. flush_cache_page(vma, vmaddr, pfn);
  285. } else {
  286. /*
  287. * For aliasing VIPT, we can flush an alias of the
  288. * userspace address only.
  289. */
  290. flush_pfn_alias(pfn, vmaddr);
  291. __flush_icache_all();
  292. }
  293. /*
  294. * Invalidate kernel mapping. No data should be contained
  295. * in this mapping of the page. FIXME: this is overkill
  296. * since we actually ask for a write-back and invalidate.
  297. */
  298. __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
  299. }