pgtable.h 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722
  1. #ifndef _ASM_GENERIC_PGTABLE_H
  2. #define _ASM_GENERIC_PGTABLE_H
  3. #ifndef __ASSEMBLY__
  4. #ifdef CONFIG_MMU
  5. #include <linux/mm_types.h>
  6. #include <linux/bug.h>
  7. /*
  8. * On almost all architectures and configurations, 0 can be used as the
  9. * upper ceiling to free_pgtables(): on many architectures it has the same
  10. * effect as using TASK_SIZE. However, there is one configuration which
  11. * must impose a more careful limit, to avoid freeing kernel pgtables.
  12. */
  13. #ifndef USER_PGTABLES_CEILING
  14. #define USER_PGTABLES_CEILING 0UL
  15. #endif
  16. #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
  17. extern int ptep_set_access_flags(struct vm_area_struct *vma,
  18. unsigned long address, pte_t *ptep,
  19. pte_t entry, int dirty);
  20. #endif
  21. #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
  22. extern int pmdp_set_access_flags(struct vm_area_struct *vma,
  23. unsigned long address, pmd_t *pmdp,
  24. pmd_t entry, int dirty);
  25. #endif
  26. #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
  27. static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
  28. unsigned long address,
  29. pte_t *ptep)
  30. {
  31. pte_t pte = *ptep;
  32. int r = 1;
  33. if (!pte_young(pte))
  34. r = 0;
  35. else
  36. set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
  37. return r;
  38. }
  39. #endif
  40. #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
  41. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  42. static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
  43. unsigned long address,
  44. pmd_t *pmdp)
  45. {
  46. pmd_t pmd = *pmdp;
  47. int r = 1;
  48. if (!pmd_young(pmd))
  49. r = 0;
  50. else
  51. set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
  52. return r;
  53. }
  54. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  55. static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
  56. unsigned long address,
  57. pmd_t *pmdp)
  58. {
  59. BUG();
  60. return 0;
  61. }
  62. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  63. #endif
  64. #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
  65. int ptep_clear_flush_young(struct vm_area_struct *vma,
  66. unsigned long address, pte_t *ptep);
  67. #endif
  68. #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
  69. int pmdp_clear_flush_young(struct vm_area_struct *vma,
  70. unsigned long address, pmd_t *pmdp);
  71. #endif
  72. #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
  73. static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
  74. unsigned long address,
  75. pte_t *ptep)
  76. {
  77. pte_t pte = *ptep;
  78. pte_clear(mm, address, ptep);
  79. return pte;
  80. }
  81. #endif
  82. #ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR
  83. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  84. static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
  85. unsigned long address,
  86. pmd_t *pmdp)
  87. {
  88. pmd_t pmd = *pmdp;
  89. pmd_clear(pmdp);
  90. return pmd;
  91. }
  92. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  93. #endif
  94. #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
  95. static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
  96. unsigned long address, pte_t *ptep,
  97. int full)
  98. {
  99. pte_t pte;
  100. pte = ptep_get_and_clear(mm, address, ptep);
  101. return pte;
  102. }
  103. #endif
  104. /*
  105. * Some architectures may be able to avoid expensive synchronization
  106. * primitives when modifications are made to PTE's which are already
  107. * not present, or in the process of an address space destruction.
  108. */
  109. #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
  110. static inline void pte_clear_not_present_full(struct mm_struct *mm,
  111. unsigned long address,
  112. pte_t *ptep,
  113. int full)
  114. {
  115. pte_clear(mm, address, ptep);
  116. }
  117. #endif
  118. #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
  119. extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
  120. unsigned long address,
  121. pte_t *ptep);
  122. #endif
  123. #ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
  124. extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
  125. unsigned long address,
  126. pmd_t *pmdp);
  127. #endif
  128. #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
  129. struct mm_struct;
  130. static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
  131. {
  132. pte_t old_pte = *ptep;
  133. set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
  134. }
  135. #endif
  136. #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
  137. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  138. static inline void pmdp_set_wrprotect(struct mm_struct *mm,
  139. unsigned long address, pmd_t *pmdp)
  140. {
  141. pmd_t old_pmd = *pmdp;
  142. set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
  143. }
  144. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  145. static inline void pmdp_set_wrprotect(struct mm_struct *mm,
  146. unsigned long address, pmd_t *pmdp)
  147. {
  148. BUG();
  149. }
  150. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  151. #endif
  152. #ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
  153. extern void pmdp_splitting_flush(struct vm_area_struct *vma,
  154. unsigned long address, pmd_t *pmdp);
  155. #endif
  156. #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
  157. extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
  158. pgtable_t pgtable);
  159. #endif
  160. #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
  161. extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
  162. #endif
  163. #ifndef __HAVE_ARCH_PMDP_INVALIDATE
  164. extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
  165. pmd_t *pmdp);
  166. #endif
  167. #ifndef __HAVE_ARCH_PTE_SAME
  168. static inline int pte_same(pte_t pte_a, pte_t pte_b)
  169. {
  170. return pte_val(pte_a) == pte_val(pte_b);
  171. }
  172. #endif
  173. #ifndef __HAVE_ARCH_PMD_SAME
  174. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  175. static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
  176. {
  177. return pmd_val(pmd_a) == pmd_val(pmd_b);
  178. }
  179. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  180. static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
  181. {
  182. BUG();
  183. return 0;
  184. }
  185. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  186. #endif
  187. #ifndef __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG
  188. #define page_test_and_clear_young(pfn) (0)
  189. #endif
  190. #ifndef __HAVE_ARCH_PGD_OFFSET_GATE
  191. #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
  192. #endif
  193. #ifndef __HAVE_ARCH_MOVE_PTE
  194. #define move_pte(pte, prot, old_addr, new_addr) (pte)
  195. #endif
  196. #ifndef pte_accessible
  197. # define pte_accessible(pte) ((void)(pte),1)
  198. #endif
  199. #ifndef flush_tlb_fix_spurious_fault
  200. #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
  201. #endif
  202. #ifndef pgprot_noncached
  203. #define pgprot_noncached(prot) (prot)
  204. #endif
  205. #ifndef pgprot_writecombine
  206. #define pgprot_writecombine pgprot_noncached
  207. #endif
  208. /*
  209. * When walking page tables, get the address of the next boundary,
  210. * or the end address of the range if that comes earlier. Although no
  211. * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout.
  212. */
  213. #define pgd_addr_end(addr, end) \
  214. ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
  215. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  216. })
  217. #ifndef pud_addr_end
  218. #define pud_addr_end(addr, end) \
  219. ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
  220. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  221. })
  222. #endif
  223. #ifndef pmd_addr_end
  224. #define pmd_addr_end(addr, end) \
  225. ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
  226. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  227. })
  228. #endif
  229. /*
  230. * When walking page tables, we usually want to skip any p?d_none entries;
  231. * and any p?d_bad entries - reporting the error before resetting to none.
  232. * Do the tests inline, but report and clear the bad entry in mm/memory.c.
  233. */
  234. void pgd_clear_bad(pgd_t *);
  235. void pud_clear_bad(pud_t *);
  236. void pmd_clear_bad(pmd_t *);
  237. static inline int pgd_none_or_clear_bad(pgd_t *pgd)
  238. {
  239. if (pgd_none(*pgd))
  240. return 1;
  241. if (unlikely(pgd_bad(*pgd))) {
  242. pgd_clear_bad(pgd);
  243. return 1;
  244. }
  245. return 0;
  246. }
  247. static inline int pud_none_or_clear_bad(pud_t *pud)
  248. {
  249. if (pud_none(*pud))
  250. return 1;
  251. if (unlikely(pud_bad(*pud))) {
  252. pud_clear_bad(pud);
  253. return 1;
  254. }
  255. return 0;
  256. }
  257. static inline int pmd_none_or_clear_bad(pmd_t *pmd)
  258. {
  259. if (pmd_none(*pmd))
  260. return 1;
  261. if (unlikely(pmd_bad(*pmd))) {
  262. pmd_clear_bad(pmd);
  263. return 1;
  264. }
  265. return 0;
  266. }
  267. static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm,
  268. unsigned long addr,
  269. pte_t *ptep)
  270. {
  271. /*
  272. * Get the current pte state, but zero it out to make it
  273. * non-present, preventing the hardware from asynchronously
  274. * updating it.
  275. */
  276. return ptep_get_and_clear(mm, addr, ptep);
  277. }
  278. static inline void __ptep_modify_prot_commit(struct mm_struct *mm,
  279. unsigned long addr,
  280. pte_t *ptep, pte_t pte)
  281. {
  282. /*
  283. * The pte is non-present, so there's no hardware state to
  284. * preserve.
  285. */
  286. set_pte_at(mm, addr, ptep, pte);
  287. }
  288. #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
  289. /*
  290. * Start a pte protection read-modify-write transaction, which
  291. * protects against asynchronous hardware modifications to the pte.
  292. * The intention is not to prevent the hardware from making pte
  293. * updates, but to prevent any updates it may make from being lost.
  294. *
  295. * This does not protect against other software modifications of the
  296. * pte; the appropriate pte lock must be held over the transation.
  297. *
  298. * Note that this interface is intended to be batchable, meaning that
  299. * ptep_modify_prot_commit may not actually update the pte, but merely
  300. * queue the update to be done at some later time. The update must be
  301. * actually committed before the pte lock is released, however.
  302. */
  303. static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
  304. unsigned long addr,
  305. pte_t *ptep)
  306. {
  307. return __ptep_modify_prot_start(mm, addr, ptep);
  308. }
  309. /*
  310. * Commit an update to a pte, leaving any hardware-controlled bits in
  311. * the PTE unmodified.
  312. */
  313. static inline void ptep_modify_prot_commit(struct mm_struct *mm,
  314. unsigned long addr,
  315. pte_t *ptep, pte_t pte)
  316. {
  317. __ptep_modify_prot_commit(mm, addr, ptep, pte);
  318. }
  319. #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
  320. #endif /* CONFIG_MMU */
  321. /*
  322. * A facility to provide lazy MMU batching. This allows PTE updates and
  323. * page invalidations to be delayed until a call to leave lazy MMU mode
  324. * is issued. Some architectures may benefit from doing this, and it is
  325. * beneficial for both shadow and direct mode hypervisors, which may batch
  326. * the PTE updates which happen during this window. Note that using this
  327. * interface requires that read hazards be removed from the code. A read
  328. * hazard could result in the direct mode hypervisor case, since the actual
  329. * write to the page tables may not yet have taken place, so reads though
  330. * a raw PTE pointer after it has been modified are not guaranteed to be
  331. * up to date. This mode can only be entered and left under the protection of
  332. * the page table locks for all page tables which may be modified. In the UP
  333. * case, this is required so that preemption is disabled, and in the SMP case,
  334. * it must synchronize the delayed page table writes properly on other CPUs.
  335. */
  336. #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
  337. #define arch_enter_lazy_mmu_mode() do {} while (0)
  338. #define arch_leave_lazy_mmu_mode() do {} while (0)
  339. #define arch_flush_lazy_mmu_mode() do {} while (0)
  340. #endif
  341. /*
  342. * A facility to provide batching of the reload of page tables and
  343. * other process state with the actual context switch code for
  344. * paravirtualized guests. By convention, only one of the batched
  345. * update (lazy) modes (CPU, MMU) should be active at any given time,
  346. * entry should never be nested, and entry and exits should always be
  347. * paired. This is for sanity of maintaining and reasoning about the
  348. * kernel code. In this case, the exit (end of the context switch) is
  349. * in architecture-specific code, and so doesn't need a generic
  350. * definition.
  351. */
  352. #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
  353. #define arch_start_context_switch(prev) do {} while (0)
  354. #endif
  355. #ifndef CONFIG_HAVE_ARCH_SOFT_DIRTY
  356. static inline int pte_soft_dirty(pte_t pte)
  357. {
  358. return 0;
  359. }
  360. static inline int pmd_soft_dirty(pmd_t pmd)
  361. {
  362. return 0;
  363. }
  364. static inline pte_t pte_mksoft_dirty(pte_t pte)
  365. {
  366. return pte;
  367. }
  368. static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
  369. {
  370. return pmd;
  371. }
  372. #endif
  373. #ifndef __HAVE_PFNMAP_TRACKING
  374. /*
  375. * Interfaces that can be used by architecture code to keep track of
  376. * memory type of pfn mappings specified by the remap_pfn_range,
  377. * vm_insert_pfn.
  378. */
  379. /*
  380. * track_pfn_remap is called when a _new_ pfn mapping is being established
  381. * by remap_pfn_range() for physical range indicated by pfn and size.
  382. */
  383. static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
  384. unsigned long pfn, unsigned long addr,
  385. unsigned long size)
  386. {
  387. return 0;
  388. }
  389. /*
  390. * track_pfn_insert is called when a _new_ single pfn is established
  391. * by vm_insert_pfn().
  392. */
  393. static inline int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
  394. unsigned long pfn)
  395. {
  396. return 0;
  397. }
  398. /*
  399. * track_pfn_copy is called when vma that is covering the pfnmap gets
  400. * copied through copy_page_range().
  401. */
  402. static inline int track_pfn_copy(struct vm_area_struct *vma)
  403. {
  404. return 0;
  405. }
  406. /*
  407. * untrack_pfn_vma is called while unmapping a pfnmap for a region.
  408. * untrack can be called for a specific region indicated by pfn and size or
  409. * can be for the entire vma (in which case pfn, size are zero).
  410. */
  411. static inline void untrack_pfn(struct vm_area_struct *vma,
  412. unsigned long pfn, unsigned long size)
  413. {
  414. }
  415. #else
  416. extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
  417. unsigned long pfn, unsigned long addr,
  418. unsigned long size);
  419. extern int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
  420. unsigned long pfn);
  421. extern int track_pfn_copy(struct vm_area_struct *vma);
  422. extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
  423. unsigned long size);
  424. #endif
  425. #ifdef __HAVE_COLOR_ZERO_PAGE
  426. static inline int is_zero_pfn(unsigned long pfn)
  427. {
  428. extern unsigned long zero_pfn;
  429. unsigned long offset_from_zero_pfn = pfn - zero_pfn;
  430. return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
  431. }
  432. #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
  433. #else
  434. static inline int is_zero_pfn(unsigned long pfn)
  435. {
  436. extern unsigned long zero_pfn;
  437. return pfn == zero_pfn;
  438. }
  439. static inline unsigned long my_zero_pfn(unsigned long addr)
  440. {
  441. extern unsigned long zero_pfn;
  442. return zero_pfn;
  443. }
  444. #endif
  445. #ifdef CONFIG_MMU
  446. #ifndef CONFIG_TRANSPARENT_HUGEPAGE
  447. static inline int pmd_trans_huge(pmd_t pmd)
  448. {
  449. return 0;
  450. }
  451. static inline int pmd_trans_splitting(pmd_t pmd)
  452. {
  453. return 0;
  454. }
  455. #ifndef __HAVE_ARCH_PMD_WRITE
  456. static inline int pmd_write(pmd_t pmd)
  457. {
  458. BUG();
  459. return 0;
  460. }
  461. #endif /* __HAVE_ARCH_PMD_WRITE */
  462. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  463. #ifndef pmd_read_atomic
  464. static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
  465. {
  466. /*
  467. * Depend on compiler for an atomic pmd read. NOTE: this is
  468. * only going to work, if the pmdval_t isn't larger than
  469. * an unsigned long.
  470. */
  471. return *pmdp;
  472. }
  473. #endif
  474. /*
  475. * This function is meant to be used by sites walking pagetables with
  476. * the mmap_sem hold in read mode to protect against MADV_DONTNEED and
  477. * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd
  478. * into a null pmd and the transhuge page fault can convert a null pmd
  479. * into an hugepmd or into a regular pmd (if the hugepage allocation
  480. * fails). While holding the mmap_sem in read mode the pmd becomes
  481. * stable and stops changing under us only if it's not null and not a
  482. * transhuge pmd. When those races occurs and this function makes a
  483. * difference vs the standard pmd_none_or_clear_bad, the result is
  484. * undefined so behaving like if the pmd was none is safe (because it
  485. * can return none anyway). The compiler level barrier() is critically
  486. * important to compute the two checks atomically on the same pmdval.
  487. *
  488. * For 32bit kernels with a 64bit large pmd_t this automatically takes
  489. * care of reading the pmd atomically to avoid SMP race conditions
  490. * against pmd_populate() when the mmap_sem is hold for reading by the
  491. * caller (a special atomic read not done by "gcc" as in the generic
  492. * version above, is also needed when THP is disabled because the page
  493. * fault can populate the pmd from under us).
  494. */
  495. static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
  496. {
  497. pmd_t pmdval = pmd_read_atomic(pmd);
  498. /*
  499. * The barrier will stabilize the pmdval in a register or on
  500. * the stack so that it will stop changing under the code.
  501. *
  502. * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE,
  503. * pmd_read_atomic is allowed to return a not atomic pmdval
  504. * (for example pointing to an hugepage that has never been
  505. * mapped in the pmd). The below checks will only care about
  506. * the low part of the pmd with 32bit PAE x86 anyway, with the
  507. * exception of pmd_none(). So the important thing is that if
  508. * the low part of the pmd is found null, the high part will
  509. * be also null or the pmd_none() check below would be
  510. * confused.
  511. */
  512. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  513. barrier();
  514. #endif
  515. if (pmd_none(pmdval))
  516. return 1;
  517. if (unlikely(pmd_bad(pmdval))) {
  518. if (!pmd_trans_huge(pmdval))
  519. pmd_clear_bad(pmd);
  520. return 1;
  521. }
  522. return 0;
  523. }
  524. /*
  525. * This is a noop if Transparent Hugepage Support is not built into
  526. * the kernel. Otherwise it is equivalent to
  527. * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in
  528. * places that already verified the pmd is not none and they want to
  529. * walk ptes while holding the mmap sem in read mode (write mode don't
  530. * need this). If THP is not enabled, the pmd can't go away under the
  531. * code even if MADV_DONTNEED runs, but if THP is enabled we need to
  532. * run a pmd_trans_unstable before walking the ptes after
  533. * split_huge_page_pmd returns (because it may have run when the pmd
  534. * become null, but then a page fault can map in a THP and not a
  535. * regular page).
  536. */
  537. static inline int pmd_trans_unstable(pmd_t *pmd)
  538. {
  539. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  540. return pmd_none_or_trans_huge_or_clear_bad(pmd);
  541. #else
  542. return 0;
  543. #endif
  544. }
  545. #ifdef CONFIG_NUMA_BALANCING
  546. #ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
  547. /*
  548. * _PAGE_NUMA works identical to _PAGE_PROTNONE (it's actually the
  549. * same bit too). It's set only when _PAGE_PRESET is not set and it's
  550. * never set if _PAGE_PRESENT is set.
  551. *
  552. * pte/pmd_present() returns true if pte/pmd_numa returns true. Page
  553. * fault triggers on those regions if pte/pmd_numa returns true
  554. * (because _PAGE_PRESENT is not set).
  555. */
  556. #ifndef pte_numa
  557. static inline int pte_numa(pte_t pte)
  558. {
  559. return (pte_flags(pte) &
  560. (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA;
  561. }
  562. #endif
  563. #ifndef pmd_numa
  564. static inline int pmd_numa(pmd_t pmd)
  565. {
  566. return (pmd_flags(pmd) &
  567. (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA;
  568. }
  569. #endif
  570. /*
  571. * pte/pmd_mknuma sets the _PAGE_ACCESSED bitflag automatically
  572. * because they're called by the NUMA hinting minor page fault. If we
  573. * wouldn't set the _PAGE_ACCESSED bitflag here, the TLB miss handler
  574. * would be forced to set it later while filling the TLB after we
  575. * return to userland. That would trigger a second write to memory
  576. * that we optimize away by setting _PAGE_ACCESSED here.
  577. */
  578. #ifndef pte_mknonnuma
  579. static inline pte_t pte_mknonnuma(pte_t pte)
  580. {
  581. pte = pte_clear_flags(pte, _PAGE_NUMA);
  582. return pte_set_flags(pte, _PAGE_PRESENT|_PAGE_ACCESSED);
  583. }
  584. #endif
  585. #ifndef pmd_mknonnuma
  586. static inline pmd_t pmd_mknonnuma(pmd_t pmd)
  587. {
  588. pmd = pmd_clear_flags(pmd, _PAGE_NUMA);
  589. return pmd_set_flags(pmd, _PAGE_PRESENT|_PAGE_ACCESSED);
  590. }
  591. #endif
  592. #ifndef pte_mknuma
  593. static inline pte_t pte_mknuma(pte_t pte)
  594. {
  595. pte = pte_set_flags(pte, _PAGE_NUMA);
  596. return pte_clear_flags(pte, _PAGE_PRESENT);
  597. }
  598. #endif
  599. #ifndef pmd_mknuma
  600. static inline pmd_t pmd_mknuma(pmd_t pmd)
  601. {
  602. pmd = pmd_set_flags(pmd, _PAGE_NUMA);
  603. return pmd_clear_flags(pmd, _PAGE_PRESENT);
  604. }
  605. #endif
  606. #else
  607. extern int pte_numa(pte_t pte);
  608. extern int pmd_numa(pmd_t pmd);
  609. extern pte_t pte_mknonnuma(pte_t pte);
  610. extern pmd_t pmd_mknonnuma(pmd_t pmd);
  611. extern pte_t pte_mknuma(pte_t pte);
  612. extern pmd_t pmd_mknuma(pmd_t pmd);
  613. #endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */
  614. #else
  615. static inline int pmd_numa(pmd_t pmd)
  616. {
  617. return 0;
  618. }
  619. static inline int pte_numa(pte_t pte)
  620. {
  621. return 0;
  622. }
  623. static inline pte_t pte_mknonnuma(pte_t pte)
  624. {
  625. return pte;
  626. }
  627. static inline pmd_t pmd_mknonnuma(pmd_t pmd)
  628. {
  629. return pmd;
  630. }
  631. static inline pte_t pte_mknuma(pte_t pte)
  632. {
  633. return pte;
  634. }
  635. static inline pmd_t pmd_mknuma(pmd_t pmd)
  636. {
  637. return pmd;
  638. }
  639. #endif /* CONFIG_NUMA_BALANCING */
  640. #endif /* CONFIG_MMU */
  641. #endif /* !__ASSEMBLY__ */
  642. #ifndef io_remap_pfn_range
  643. #define io_remap_pfn_range remap_pfn_range
  644. #endif
  645. #endif /* _ASM_GENERIC_PGTABLE_H */