tlbflush.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. /*
  2. * arch/arm/include/asm/tlbflush.h
  3. *
  4. * Copyright (C) 1999-2003 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #ifndef _ASMARM_TLBFLUSH_H
  11. #define _ASMARM_TLBFLUSH_H
  12. #ifdef CONFIG_MMU
  13. #include <asm/glue.h>
  14. #define TLB_V3_PAGE (1 << 0)
  15. #define TLB_V4_U_PAGE (1 << 1)
  16. #define TLB_V4_D_PAGE (1 << 2)
  17. #define TLB_V4_I_PAGE (1 << 3)
  18. #define TLB_V6_U_PAGE (1 << 4)
  19. #define TLB_V6_D_PAGE (1 << 5)
  20. #define TLB_V6_I_PAGE (1 << 6)
  21. #define TLB_V3_FULL (1 << 8)
  22. #define TLB_V4_U_FULL (1 << 9)
  23. #define TLB_V4_D_FULL (1 << 10)
  24. #define TLB_V4_I_FULL (1 << 11)
  25. #define TLB_V6_U_FULL (1 << 12)
  26. #define TLB_V6_D_FULL (1 << 13)
  27. #define TLB_V6_I_FULL (1 << 14)
  28. #define TLB_V6_U_ASID (1 << 16)
  29. #define TLB_V6_D_ASID (1 << 17)
  30. #define TLB_V6_I_ASID (1 << 18)
  31. /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
  32. #define TLB_V7_UIS_PAGE (1 << 19)
  33. #define TLB_V7_UIS_FULL (1 << 20)
  34. #define TLB_V7_UIS_ASID (1 << 21)
  35. #define TLB_BARRIER (1 << 28)
  36. #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
  37. #define TLB_DCLEAN (1 << 30)
  38. #define TLB_WB (1 << 31)
  39. /*
  40. * MMU TLB Model
  41. * =============
  42. *
  43. * We have the following to choose from:
  44. * v3 - ARMv3
  45. * v4 - ARMv4 without write buffer
  46. * v4wb - ARMv4 with write buffer without I TLB flush entry instruction
  47. * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
  48. * fr - Feroceon (v4wbi with non-outer-cacheable page table walks)
  49. * fa - Faraday (v4 with write buffer with UTLB)
  50. * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction
  51. * v7wbi - identical to v6wbi
  52. */
  53. #undef _TLB
  54. #undef MULTI_TLB
  55. #ifdef CONFIG_SMP_ON_UP
  56. #define MULTI_TLB 1
  57. #endif
  58. #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE)
  59. #ifdef CONFIG_CPU_TLB_V4WT
  60. # define v4_possible_flags v4_tlb_flags
  61. # define v4_always_flags v4_tlb_flags
  62. # ifdef _TLB
  63. # define MULTI_TLB 1
  64. # else
  65. # define _TLB v4
  66. # endif
  67. #else
  68. # define v4_possible_flags 0
  69. # define v4_always_flags (-1UL)
  70. #endif
  71. #define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
  72. TLB_V4_U_FULL | TLB_V4_U_PAGE)
  73. #ifdef CONFIG_CPU_TLB_FA
  74. # define fa_possible_flags fa_tlb_flags
  75. # define fa_always_flags fa_tlb_flags
  76. # ifdef _TLB
  77. # define MULTI_TLB 1
  78. # else
  79. # define _TLB fa
  80. # endif
  81. #else
  82. # define fa_possible_flags 0
  83. # define fa_always_flags (-1UL)
  84. #endif
  85. #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \
  86. TLB_V4_I_FULL | TLB_V4_D_FULL | \
  87. TLB_V4_I_PAGE | TLB_V4_D_PAGE)
  88. #ifdef CONFIG_CPU_TLB_V4WBI
  89. # define v4wbi_possible_flags v4wbi_tlb_flags
  90. # define v4wbi_always_flags v4wbi_tlb_flags
  91. # ifdef _TLB
  92. # define MULTI_TLB 1
  93. # else
  94. # define _TLB v4wbi
  95. # endif
  96. #else
  97. # define v4wbi_possible_flags 0
  98. # define v4wbi_always_flags (-1UL)
  99. #endif
  100. #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \
  101. TLB_V4_I_FULL | TLB_V4_D_FULL | \
  102. TLB_V4_I_PAGE | TLB_V4_D_PAGE)
  103. #ifdef CONFIG_CPU_TLB_FEROCEON
  104. # define fr_possible_flags fr_tlb_flags
  105. # define fr_always_flags fr_tlb_flags
  106. # ifdef _TLB
  107. # define MULTI_TLB 1
  108. # else
  109. # define _TLB v4wbi
  110. # endif
  111. #else
  112. # define fr_possible_flags 0
  113. # define fr_always_flags (-1UL)
  114. #endif
  115. #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \
  116. TLB_V4_I_FULL | TLB_V4_D_FULL | \
  117. TLB_V4_D_PAGE)
  118. #ifdef CONFIG_CPU_TLB_V4WB
  119. # define v4wb_possible_flags v4wb_tlb_flags
  120. # define v4wb_always_flags v4wb_tlb_flags
  121. # ifdef _TLB
  122. # define MULTI_TLB 1
  123. # else
  124. # define _TLB v4wb
  125. # endif
  126. #else
  127. # define v4wb_possible_flags 0
  128. # define v4wb_always_flags (-1UL)
  129. #endif
  130. #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
  131. TLB_V6_I_FULL | TLB_V6_D_FULL | \
  132. TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
  133. TLB_V6_I_ASID | TLB_V6_D_ASID)
  134. #ifdef CONFIG_CPU_TLB_V6
  135. # define v6wbi_possible_flags v6wbi_tlb_flags
  136. # define v6wbi_always_flags v6wbi_tlb_flags
  137. # ifdef _TLB
  138. # define MULTI_TLB 1
  139. # else
  140. # define _TLB v6wbi
  141. # endif
  142. #else
  143. # define v6wbi_possible_flags 0
  144. # define v6wbi_always_flags (-1UL)
  145. #endif
  146. #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
  147. TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
  148. #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
  149. TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
  150. #ifdef CONFIG_CPU_TLB_V7
  151. # ifdef CONFIG_SMP_ON_UP
  152. # define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up)
  153. # define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up)
  154. # elif defined(CONFIG_SMP)
  155. # define v7wbi_possible_flags v7wbi_tlb_flags_smp
  156. # define v7wbi_always_flags v7wbi_tlb_flags_smp
  157. # else
  158. # define v7wbi_possible_flags v7wbi_tlb_flags_up
  159. # define v7wbi_always_flags v7wbi_tlb_flags_up
  160. # endif
  161. # ifdef _TLB
  162. # define MULTI_TLB 1
  163. # else
  164. # define _TLB v7wbi
  165. # endif
  166. #else
  167. # define v7wbi_possible_flags 0
  168. # define v7wbi_always_flags (-1UL)
  169. #endif
  170. #ifndef _TLB
  171. #error Unknown TLB model
  172. #endif
  173. #ifndef __ASSEMBLY__
  174. #include <linux/sched.h>
  175. struct cpu_tlb_fns {
  176. void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
  177. void (*flush_kern_range)(unsigned long, unsigned long);
  178. unsigned long tlb_flags;
  179. };
  180. /*
  181. * Select the calling method
  182. */
  183. #ifdef MULTI_TLB
  184. #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range
  185. #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range
  186. #else
  187. #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range)
  188. #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range)
  189. extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  190. extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
  191. #endif
  192. extern struct cpu_tlb_fns cpu_tlb;
  193. #define __cpu_tlb_flags cpu_tlb.tlb_flags
  194. /*
  195. * TLB Management
  196. * ==============
  197. *
  198. * The arch/arm/mm/tlb-*.S files implement these methods.
  199. *
  200. * The TLB specific code is expected to perform whatever tests it
  201. * needs to determine if it should invalidate the TLB for each
  202. * call. Start addresses are inclusive and end addresses are
  203. * exclusive; it is safe to round these addresses down.
  204. *
  205. * flush_tlb_all()
  206. *
  207. * Invalidate the entire TLB.
  208. *
  209. * flush_tlb_mm(mm)
  210. *
  211. * Invalidate all TLB entries in a particular address
  212. * space.
  213. * - mm - mm_struct describing address space
  214. *
  215. * flush_tlb_range(mm,start,end)
  216. *
  217. * Invalidate a range of TLB entries in the specified
  218. * address space.
  219. * - mm - mm_struct describing address space
  220. * - start - start address (may not be aligned)
  221. * - end - end address (exclusive, may not be aligned)
  222. *
  223. * flush_tlb_page(vaddr,vma)
  224. *
  225. * Invalidate the specified page in the specified address range.
  226. * - vaddr - virtual address (may not be aligned)
  227. * - vma - vma_struct describing address range
  228. *
  229. * flush_kern_tlb_page(kaddr)
  230. *
  231. * Invalidate the TLB entry for the specified page. The address
  232. * will be in the kernels virtual memory space. Current uses
  233. * only require the D-TLB to be invalidated.
  234. * - kaddr - Kernel virtual memory address
  235. */
  236. /*
  237. * We optimise the code below by:
  238. * - building a set of TLB flags that might be set in __cpu_tlb_flags
  239. * - building a set of TLB flags that will always be set in __cpu_tlb_flags
  240. * - if we're going to need __cpu_tlb_flags, access it once and only once
  241. *
  242. * This allows us to build optimal assembly for the single-CPU type case,
  243. * and as close to optimal given the compiler constrants for multi-CPU
  244. * case. We could do better for the multi-CPU case if the compiler
  245. * implemented the "%?" method, but this has been discontinued due to too
  246. * many people getting it wrong.
  247. */
  248. #define possible_tlb_flags (v4_possible_flags | \
  249. v4wbi_possible_flags | \
  250. fr_possible_flags | \
  251. v4wb_possible_flags | \
  252. fa_possible_flags | \
  253. v6wbi_possible_flags | \
  254. v7wbi_possible_flags)
  255. #define always_tlb_flags (v4_always_flags & \
  256. v4wbi_always_flags & \
  257. fr_always_flags & \
  258. v4wb_always_flags & \
  259. fa_always_flags & \
  260. v6wbi_always_flags & \
  261. v7wbi_always_flags)
  262. #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
  263. #define __tlb_op(f, insnarg, arg) \
  264. do { \
  265. if (always_tlb_flags & (f)) \
  266. asm("mcr " insnarg \
  267. : : "r" (arg) : "cc"); \
  268. else if (possible_tlb_flags & (f)) \
  269. asm("tst %1, %2\n\t" \
  270. "mcrne " insnarg \
  271. : : "r" (arg), "r" (__tlb_flag), "Ir" (f) \
  272. : "cc"); \
  273. } while (0)
  274. #define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg)
  275. #define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg)
  276. static inline void local_flush_tlb_all(void)
  277. {
  278. const int zero = 0;
  279. const unsigned int __tlb_flag = __cpu_tlb_flags;
  280. if (tlb_flag(TLB_WB))
  281. dsb();
  282. tlb_op(TLB_V3_FULL, "c6, c0, 0", zero);
  283. tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero);
  284. tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero);
  285. tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
  286. tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
  287. if (tlb_flag(TLB_BARRIER)) {
  288. dsb();
  289. isb();
  290. }
  291. }
  292. static inline void local_flush_tlb_mm(struct mm_struct *mm)
  293. {
  294. const int zero = 0;
  295. const int asid = ASID(mm);
  296. const unsigned int __tlb_flag = __cpu_tlb_flags;
  297. if (tlb_flag(TLB_WB))
  298. dsb();
  299. if (possible_tlb_flags & (TLB_V3_FULL|TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) {
  300. if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) {
  301. tlb_op(TLB_V3_FULL, "c6, c0, 0", zero);
  302. tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero);
  303. tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero);
  304. tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero);
  305. }
  306. put_cpu();
  307. }
  308. tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid);
  309. tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid);
  310. tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid);
  311. #ifdef CONFIG_ARM_ERRATA_720789
  312. tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", zero);
  313. #else
  314. tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", asid);
  315. #endif
  316. if (tlb_flag(TLB_BARRIER))
  317. dsb();
  318. }
  319. static inline void
  320. local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
  321. {
  322. const int zero = 0;
  323. const unsigned int __tlb_flag = __cpu_tlb_flags;
  324. uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
  325. if (tlb_flag(TLB_WB))
  326. dsb();
  327. if (possible_tlb_flags & (TLB_V3_PAGE|TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) &&
  328. cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
  329. tlb_op(TLB_V3_PAGE, "c6, c0, 0", uaddr);
  330. tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr);
  331. tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr);
  332. tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr);
  333. if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
  334. asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
  335. }
  336. tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr);
  337. tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr);
  338. tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr);
  339. #ifdef CONFIG_ARM_ERRATA_720789
  340. tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK);
  341. #else
  342. tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr);
  343. #endif
  344. if (tlb_flag(TLB_BARRIER))
  345. dsb();
  346. }
  347. static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
  348. {
  349. const int zero = 0;
  350. const unsigned int __tlb_flag = __cpu_tlb_flags;
  351. kaddr &= PAGE_MASK;
  352. if (tlb_flag(TLB_WB))
  353. dsb();
  354. tlb_op(TLB_V3_PAGE, "c6, c0, 0", kaddr);
  355. tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr);
  356. tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr);
  357. tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr);
  358. if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
  359. asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
  360. tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr);
  361. tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr);
  362. tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr);
  363. tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
  364. if (tlb_flag(TLB_BARRIER)) {
  365. dsb();
  366. isb();
  367. }
  368. }
  369. /*
  370. * flush_pmd_entry
  371. *
  372. * Flush a PMD entry (word aligned, or double-word aligned) to
  373. * RAM if the TLB for the CPU we are running on requires this.
  374. * This is typically used when we are creating PMD entries.
  375. *
  376. * clean_pmd_entry
  377. *
  378. * Clean (but don't drain the write buffer) if the CPU requires
  379. * these operations. This is typically used when we are removing
  380. * PMD entries.
  381. */
  382. static inline void flush_pmd_entry(void *pmd)
  383. {
  384. const unsigned int __tlb_flag = __cpu_tlb_flags;
  385. tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd);
  386. tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
  387. if (tlb_flag(TLB_WB))
  388. dsb();
  389. }
  390. static inline void clean_pmd_entry(void *pmd)
  391. {
  392. const unsigned int __tlb_flag = __cpu_tlb_flags;
  393. tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd);
  394. tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd);
  395. }
  396. #undef tlb_op
  397. #undef tlb_flag
  398. #undef always_tlb_flags
  399. #undef possible_tlb_flags
  400. /*
  401. * Convert calls to our calling convention.
  402. */
  403. #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
  404. #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
  405. #ifndef CONFIG_SMP
  406. #define flush_tlb_all local_flush_tlb_all
  407. #define flush_tlb_mm local_flush_tlb_mm
  408. #define flush_tlb_page local_flush_tlb_page
  409. #define flush_tlb_kernel_page local_flush_tlb_kernel_page
  410. #define flush_tlb_range local_flush_tlb_range
  411. #define flush_tlb_kernel_range local_flush_tlb_kernel_range
  412. #else
  413. extern void flush_tlb_all(void);
  414. extern void flush_tlb_mm(struct mm_struct *mm);
  415. extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);
  416. extern void flush_tlb_kernel_page(unsigned long kaddr);
  417. extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  418. extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
  419. #endif
  420. /*
  421. * If PG_dcache_clean is not set for the page, we need to ensure that any
  422. * cache entries for the kernels virtual memory range are written
  423. * back to the page. On ARMv6 and later, the cache coherency is handled via
  424. * the set_pte_at() function.
  425. */
  426. #if __LINUX_ARM_ARCH__ < 6
  427. extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
  428. pte_t *ptep);
  429. #else
  430. static inline void update_mmu_cache(struct vm_area_struct *vma,
  431. unsigned long addr, pte_t *ptep)
  432. {
  433. }
  434. #endif
  435. #endif
  436. #endif /* CONFIG_MMU */
  437. #endif