mm.h 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781
  1. #ifndef _LINUX_MM_H
  2. #define _LINUX_MM_H
  3. #include <linux/errno.h>
  4. #ifdef __KERNEL__
  5. #include <linux/gfp.h>
  6. #include <linux/bug.h>
  7. #include <linux/list.h>
  8. #include <linux/mmzone.h>
  9. #include <linux/rbtree.h>
  10. #include <linux/atomic.h>
  11. #include <linux/debug_locks.h>
  12. #include <linux/mm_types.h>
  13. #include <linux/range.h>
  14. #include <linux/pfn.h>
  15. #include <linux/bit_spinlock.h>
  16. #include <linux/shrinker.h>
  17. struct mempolicy;
  18. struct anon_vma;
  19. struct anon_vma_chain;
  20. struct file_ra_state;
  21. struct user_struct;
  22. struct writeback_control;
  23. #ifndef CONFIG_DISCONTIGMEM /* Don't use mapnrs, do it properly */
  24. extern unsigned long max_mapnr;
  25. #endif
  26. extern unsigned long num_physpages;
  27. extern unsigned long totalram_pages;
  28. extern void * high_memory;
  29. extern int page_cluster;
  30. #ifdef CONFIG_SYSCTL
  31. extern int sysctl_legacy_va_layout;
  32. #else
  33. #define sysctl_legacy_va_layout 0
  34. #endif
  35. #include <asm/page.h>
  36. #include <asm/pgtable.h>
  37. #include <asm/processor.h>
  38. #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
  39. /* to align the pointer to the (next) page boundary */
  40. #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
  41. /*
  42. * Linux kernel virtual memory manager primitives.
  43. * The idea being to have a "virtual" mm in the same way
  44. * we have a virtual fs - giving a cleaner interface to the
  45. * mm details, and allowing different kinds of memory mappings
  46. * (from shared memory to executable loading to arbitrary
  47. * mmap() functions).
  48. */
  49. extern struct kmem_cache *vm_area_cachep;
  50. #ifndef CONFIG_MMU
  51. extern struct rb_root nommu_region_tree;
  52. extern struct rw_semaphore nommu_region_sem;
  53. extern unsigned int kobjsize(const void *objp);
  54. #endif
  55. /*
  56. * vm_flags in vm_area_struct, see mm_types.h.
  57. */
  58. #define VM_NONE 0x00000000
  59. #define VM_READ 0x00000001 /* currently active flags */
  60. #define VM_WRITE 0x00000002
  61. #define VM_EXEC 0x00000004
  62. #define VM_SHARED 0x00000008
  63. /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
  64. #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
  65. #define VM_MAYWRITE 0x00000020
  66. #define VM_MAYEXEC 0x00000040
  67. #define VM_MAYSHARE 0x00000080
  68. #define VM_GROWSDOWN 0x00000100 /* general info on the segment */
  69. #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
  70. #define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
  71. #define VM_POPULATE 0x00001000
  72. #define VM_LOCKED 0x00002000
  73. #define VM_IO 0x00004000 /* Memory mapped I/O or similar */
  74. /* Used by sys_madvise() */
  75. #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
  76. #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
  77. #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
  78. #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
  79. #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
  80. #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
  81. #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
  82. #define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */
  83. #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
  84. #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
  85. #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
  86. #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
  87. #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
  88. #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
  89. #if defined(CONFIG_X86)
  90. # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
  91. #elif defined(CONFIG_PPC)
  92. # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
  93. #elif defined(CONFIG_PARISC)
  94. # define VM_GROWSUP VM_ARCH_1
  95. #elif defined(CONFIG_IA64)
  96. # define VM_GROWSUP VM_ARCH_1
  97. #elif !defined(CONFIG_MMU)
  98. # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
  99. #endif
  100. #ifndef VM_GROWSUP
  101. # define VM_GROWSUP VM_NONE
  102. #endif
  103. /* Bits set in the VMA until the stack is in its final location */
  104. #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
  105. #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
  106. #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
  107. #endif
  108. #ifdef CONFIG_STACK_GROWSUP
  109. #define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  110. #else
  111. #define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  112. #endif
  113. #define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
  114. #define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
  115. #define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
  116. #define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
  117. #define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
  118. /*
  119. * Special vmas that are non-mergable, non-mlock()able.
  120. * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
  121. */
  122. #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)
  123. /*
  124. * mapping from the currently active vm_flags protection bits (the
  125. * low four bits) to a page protection mask..
  126. */
  127. extern pgprot_t protection_map[16];
  128. #define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
  129. #define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */
  130. #define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */
  131. #define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */
  132. #define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */
  133. #define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */
  134. #define FAULT_FLAG_TRIED 0x40 /* second try */
  135. /*
  136. * vm_fault is filled by the the pagefault handler and passed to the vma's
  137. * ->fault function. The vma's ->fault is responsible for returning a bitmask
  138. * of VM_FAULT_xxx flags that give details about how the fault was handled.
  139. *
  140. * pgoff should be used in favour of virtual_address, if possible. If pgoff
  141. * is used, one may implement ->remap_pages to get nonlinear mapping support.
  142. */
  143. struct vm_fault {
  144. unsigned int flags; /* FAULT_FLAG_xxx flags */
  145. pgoff_t pgoff; /* Logical page offset based on vma */
  146. void __user *virtual_address; /* Faulting virtual address */
  147. struct page *page; /* ->fault handlers should return a
  148. * page here, unless VM_FAULT_NOPAGE
  149. * is set (which is also implied by
  150. * VM_FAULT_ERROR).
  151. */
  152. };
  153. /*
  154. * These are the virtual MM functions - opening of an area, closing and
  155. * unmapping it (needed to keep files on disk up-to-date etc), pointer
  156. * to the functions called when a no-page or a wp-page exception occurs.
  157. */
  158. struct vm_operations_struct {
  159. void (*open)(struct vm_area_struct * area);
  160. void (*close)(struct vm_area_struct * area);
  161. int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
  162. /* notification that a previously read-only page is about to become
  163. * writable, if an error is returned it will cause a SIGBUS */
  164. int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
  165. /* called by access_process_vm when get_user_pages() fails, typically
  166. * for use by special VMAs that can switch between memory and hardware
  167. */
  168. int (*access)(struct vm_area_struct *vma, unsigned long addr,
  169. void *buf, int len, int write);
  170. #ifdef CONFIG_NUMA
  171. /*
  172. * set_policy() op must add a reference to any non-NULL @new mempolicy
  173. * to hold the policy upon return. Caller should pass NULL @new to
  174. * remove a policy and fall back to surrounding context--i.e. do not
  175. * install a MPOL_DEFAULT policy, nor the task or system default
  176. * mempolicy.
  177. */
  178. int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
  179. /*
  180. * get_policy() op must add reference [mpol_get()] to any policy at
  181. * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
  182. * in mm/mempolicy.c will do this automatically.
  183. * get_policy() must NOT add a ref if the policy at (vma,addr) is not
  184. * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
  185. * If no [shared/vma] mempolicy exists at the addr, get_policy() op
  186. * must return NULL--i.e., do not "fallback" to task or system default
  187. * policy.
  188. */
  189. struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
  190. unsigned long addr);
  191. int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
  192. const nodemask_t *to, unsigned long flags);
  193. #endif
  194. /* called by sys_remap_file_pages() to populate non-linear mapping */
  195. int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
  196. unsigned long size, pgoff_t pgoff);
  197. };
  198. struct mmu_gather;
  199. struct inode;
  200. #define page_private(page) ((page)->private)
  201. #define set_page_private(page, v) ((page)->private = (v))
  202. /* It's valid only if the page is free path or free_list */
  203. static inline void set_freepage_migratetype(struct page *page, int migratetype)
  204. {
  205. page->index = migratetype;
  206. }
  207. /* It's valid only if the page is free path or free_list */
  208. static inline int get_freepage_migratetype(struct page *page)
  209. {
  210. return page->index;
  211. }
  212. /*
  213. * FIXME: take this include out, include page-flags.h in
  214. * files which need it (119 of them)
  215. */
  216. #include <linux/page-flags.h>
  217. #include <linux/huge_mm.h>
  218. /*
  219. * Methods to modify the page usage count.
  220. *
  221. * What counts for a page usage:
  222. * - cache mapping (page->mapping)
  223. * - private data (page->private)
  224. * - page mapped in a task's page tables, each mapping
  225. * is counted separately
  226. *
  227. * Also, many kernel routines increase the page count before a critical
  228. * routine so they can be sure the page doesn't go away from under them.
  229. */
  230. /*
  231. * Drop a ref, return true if the refcount fell to zero (the page has no users)
  232. */
  233. static inline int put_page_testzero(struct page *page)
  234. {
  235. VM_BUG_ON(atomic_read(&page->_count) == 0);
  236. return atomic_dec_and_test(&page->_count);
  237. }
  238. /*
  239. * Try to grab a ref unless the page has a refcount of zero, return false if
  240. * that is the case.
  241. */
  242. static inline int get_page_unless_zero(struct page *page)
  243. {
  244. return atomic_inc_not_zero(&page->_count);
  245. }
  246. extern int page_is_ram(unsigned long pfn);
  247. /* Support for virtually mapped pages */
  248. struct page *vmalloc_to_page(const void *addr);
  249. unsigned long vmalloc_to_pfn(const void *addr);
  250. /*
  251. * Determine if an address is within the vmalloc range
  252. *
  253. * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
  254. * is no special casing required.
  255. */
  256. static inline int is_vmalloc_addr(const void *x)
  257. {
  258. #ifdef CONFIG_MMU
  259. unsigned long addr = (unsigned long)x;
  260. return addr >= VMALLOC_START && addr < VMALLOC_END;
  261. #else
  262. return 0;
  263. #endif
  264. }
  265. #ifdef CONFIG_MMU
  266. extern int is_vmalloc_or_module_addr(const void *x);
  267. #else
  268. static inline int is_vmalloc_or_module_addr(const void *x)
  269. {
  270. return 0;
  271. }
  272. #endif
  273. static inline void compound_lock(struct page *page)
  274. {
  275. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  276. VM_BUG_ON(PageSlab(page));
  277. bit_spin_lock(PG_compound_lock, &page->flags);
  278. #endif
  279. }
  280. static inline void compound_unlock(struct page *page)
  281. {
  282. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  283. VM_BUG_ON(PageSlab(page));
  284. bit_spin_unlock(PG_compound_lock, &page->flags);
  285. #endif
  286. }
  287. static inline unsigned long compound_lock_irqsave(struct page *page)
  288. {
  289. unsigned long uninitialized_var(flags);
  290. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  291. local_irq_save(flags);
  292. compound_lock(page);
  293. #endif
  294. return flags;
  295. }
  296. static inline void compound_unlock_irqrestore(struct page *page,
  297. unsigned long flags)
  298. {
  299. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  300. compound_unlock(page);
  301. local_irq_restore(flags);
  302. #endif
  303. }
  304. static inline struct page *compound_head(struct page *page)
  305. {
  306. if (unlikely(PageTail(page)))
  307. return page->first_page;
  308. return page;
  309. }
  310. /*
  311. * The atomic page->_mapcount, starts from -1: so that transitions
  312. * both from it and to it can be tracked, using atomic_inc_and_test
  313. * and atomic_add_negative(-1).
  314. */
  315. static inline void reset_page_mapcount(struct page *page)
  316. {
  317. atomic_set(&(page)->_mapcount, -1);
  318. }
  319. static inline int page_mapcount(struct page *page)
  320. {
  321. return atomic_read(&(page)->_mapcount) + 1;
  322. }
  323. static inline int page_count(struct page *page)
  324. {
  325. return atomic_read(&compound_head(page)->_count);
  326. }
  327. static inline void get_huge_page_tail(struct page *page)
  328. {
  329. /*
  330. * __split_huge_page_refcount() cannot run
  331. * from under us.
  332. */
  333. VM_BUG_ON(page_mapcount(page) < 0);
  334. VM_BUG_ON(atomic_read(&page->_count) != 0);
  335. atomic_inc(&page->_mapcount);
  336. }
  337. extern bool __get_page_tail(struct page *page);
  338. static inline void get_page(struct page *page)
  339. {
  340. if (unlikely(PageTail(page)))
  341. if (likely(__get_page_tail(page)))
  342. return;
  343. /*
  344. * Getting a normal page or the head of a compound page
  345. * requires to already have an elevated page->_count.
  346. */
  347. VM_BUG_ON(atomic_read(&page->_count) <= 0);
  348. atomic_inc(&page->_count);
  349. }
  350. static inline struct page *virt_to_head_page(const void *x)
  351. {
  352. struct page *page = virt_to_page(x);
  353. return compound_head(page);
  354. }
  355. /*
  356. * Setup the page count before being freed into the page allocator for
  357. * the first time (boot or memory hotplug)
  358. */
  359. static inline void init_page_count(struct page *page)
  360. {
  361. atomic_set(&page->_count, 1);
  362. }
  363. /*
  364. * PageBuddy() indicate that the page is free and in the buddy system
  365. * (see mm/page_alloc.c).
  366. *
  367. * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
  368. * -2 so that an underflow of the page_mapcount() won't be mistaken
  369. * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
  370. * efficiently by most CPU architectures.
  371. */
  372. #define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
  373. static inline int PageBuddy(struct page *page)
  374. {
  375. return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
  376. }
  377. static inline void __SetPageBuddy(struct page *page)
  378. {
  379. VM_BUG_ON(atomic_read(&page->_mapcount) != -1);
  380. atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
  381. }
  382. static inline void __ClearPageBuddy(struct page *page)
  383. {
  384. VM_BUG_ON(!PageBuddy(page));
  385. atomic_set(&page->_mapcount, -1);
  386. }
  387. void put_page(struct page *page);
  388. void put_pages_list(struct list_head *pages);
  389. void split_page(struct page *page, unsigned int order);
  390. int split_free_page(struct page *page);
  391. /*
  392. * Compound pages have a destructor function. Provide a
  393. * prototype for that function and accessor functions.
  394. * These are _only_ valid on the head of a PG_compound page.
  395. */
  396. typedef void compound_page_dtor(struct page *);
  397. static inline void set_compound_page_dtor(struct page *page,
  398. compound_page_dtor *dtor)
  399. {
  400. page[1].lru.next = (void *)dtor;
  401. }
  402. static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
  403. {
  404. return (compound_page_dtor *)page[1].lru.next;
  405. }
  406. static inline int compound_order(struct page *page)
  407. {
  408. if (!PageHead(page))
  409. return 0;
  410. return (unsigned long)page[1].lru.prev;
  411. }
  412. static inline int compound_trans_order(struct page *page)
  413. {
  414. int order;
  415. unsigned long flags;
  416. if (!PageHead(page))
  417. return 0;
  418. flags = compound_lock_irqsave(page);
  419. order = compound_order(page);
  420. compound_unlock_irqrestore(page, flags);
  421. return order;
  422. }
  423. static inline void set_compound_order(struct page *page, unsigned long order)
  424. {
  425. page[1].lru.prev = (void *)order;
  426. }
  427. #ifdef CONFIG_MMU
  428. /*
  429. * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
  430. * servicing faults for write access. In the normal case, do always want
  431. * pte_mkwrite. But get_user_pages can cause write faults for mappings
  432. * that do not have writing enabled, when used by access_process_vm.
  433. */
  434. static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  435. {
  436. if (likely(vma->vm_flags & VM_WRITE))
  437. pte = pte_mkwrite(pte);
  438. return pte;
  439. }
  440. #endif
  441. /*
  442. * Multiple processes may "see" the same page. E.g. for untouched
  443. * mappings of /dev/null, all processes see the same page full of
  444. * zeroes, and text pages of executables and shared libraries have
  445. * only one copy in memory, at most, normally.
  446. *
  447. * For the non-reserved pages, page_count(page) denotes a reference count.
  448. * page_count() == 0 means the page is free. page->lru is then used for
  449. * freelist management in the buddy allocator.
  450. * page_count() > 0 means the page has been allocated.
  451. *
  452. * Pages are allocated by the slab allocator in order to provide memory
  453. * to kmalloc and kmem_cache_alloc. In this case, the management of the
  454. * page, and the fields in 'struct page' are the responsibility of mm/slab.c
  455. * unless a particular usage is carefully commented. (the responsibility of
  456. * freeing the kmalloc memory is the caller's, of course).
  457. *
  458. * A page may be used by anyone else who does a __get_free_page().
  459. * In this case, page_count still tracks the references, and should only
  460. * be used through the normal accessor functions. The top bits of page->flags
  461. * and page->virtual store page management information, but all other fields
  462. * are unused and could be used privately, carefully. The management of this
  463. * page is the responsibility of the one who allocated it, and those who have
  464. * subsequently been given references to it.
  465. *
  466. * The other pages (we may call them "pagecache pages") are completely
  467. * managed by the Linux memory manager: I/O, buffers, swapping etc.
  468. * The following discussion applies only to them.
  469. *
  470. * A pagecache page contains an opaque `private' member, which belongs to the
  471. * page's address_space. Usually, this is the address of a circular list of
  472. * the page's disk buffers. PG_private must be set to tell the VM to call
  473. * into the filesystem to release these pages.
  474. *
  475. * A page may belong to an inode's memory mapping. In this case, page->mapping
  476. * is the pointer to the inode, and page->index is the file offset of the page,
  477. * in units of PAGE_CACHE_SIZE.
  478. *
  479. * If pagecache pages are not associated with an inode, they are said to be
  480. * anonymous pages. These may become associated with the swapcache, and in that
  481. * case PG_swapcache is set, and page->private is an offset into the swapcache.
  482. *
  483. * In either case (swapcache or inode backed), the pagecache itself holds one
  484. * reference to the page. Setting PG_private should also increment the
  485. * refcount. The each user mapping also has a reference to the page.
  486. *
  487. * The pagecache pages are stored in a per-mapping radix tree, which is
  488. * rooted at mapping->page_tree, and indexed by offset.
  489. * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
  490. * lists, we instead now tag pages as dirty/writeback in the radix tree.
  491. *
  492. * All pagecache pages may be subject to I/O:
  493. * - inode pages may need to be read from disk,
  494. * - inode pages which have been modified and are MAP_SHARED may need
  495. * to be written back to the inode on disk,
  496. * - anonymous pages (including MAP_PRIVATE file mappings) which have been
  497. * modified may need to be swapped out to swap space and (later) to be read
  498. * back into memory.
  499. */
  500. /*
  501. * The zone field is never updated after free_area_init_core()
  502. * sets it, so none of the operations on it need to be atomic.
  503. */
  504. /*
  505. * page->flags layout:
  506. *
  507. * There are three possibilities for how page->flags get
  508. * laid out. The first is for the normal case, without
  509. * sparsemem. The second is for sparsemem when there is
  510. * plenty of space for node and section. The last is when
  511. * we have run out of space and have to fall back to an
  512. * alternate (slower) way of determining the node.
  513. *
  514. * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
  515. * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
  516. * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
  517. */
  518. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  519. #define SECTIONS_WIDTH SECTIONS_SHIFT
  520. #else
  521. #define SECTIONS_WIDTH 0
  522. #endif
  523. #define ZONES_WIDTH ZONES_SHIFT
  524. #if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
  525. #define NODES_WIDTH NODES_SHIFT
  526. #else
  527. #ifdef CONFIG_SPARSEMEM_VMEMMAP
  528. #error "Vmemmap: No space for nodes field in page flags"
  529. #endif
  530. #define NODES_WIDTH 0
  531. #endif
  532. /* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */
  533. #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
  534. #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
  535. #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
  536. /*
  537. * We are going to use the flags for the page to node mapping if its in
  538. * there. This includes the case where there is no node, so it is implicit.
  539. */
  540. #if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
  541. #define NODE_NOT_IN_PAGE_FLAGS
  542. #endif
  543. /*
  544. * Define the bit shifts to access each section. For non-existent
  545. * sections we define the shift as 0; that plus a 0 mask ensures
  546. * the compiler will optimise away reference to them.
  547. */
  548. #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
  549. #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
  550. #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
  551. /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
  552. #ifdef NODE_NOT_IN_PAGE_FLAGS
  553. #define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
  554. #define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
  555. SECTIONS_PGOFF : ZONES_PGOFF)
  556. #else
  557. #define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
  558. #define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
  559. NODES_PGOFF : ZONES_PGOFF)
  560. #endif
  561. #define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
  562. #if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  563. #error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  564. #endif
  565. #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
  566. #define NODES_MASK ((1UL << NODES_WIDTH) - 1)
  567. #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
  568. #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
  569. static inline enum zone_type page_zonenum(const struct page *page)
  570. {
  571. return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
  572. }
  573. /*
  574. * The identification function is only used by the buddy allocator for
  575. * determining if two pages could be buddies. We are not really
  576. * identifying a zone since we could be using a the section number
  577. * id if we have not node id available in page flags.
  578. * We guarantee only that it will return the same value for two
  579. * combinable pages in a zone.
  580. */
  581. static inline int page_zone_id(struct page *page)
  582. {
  583. return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
  584. }
  585. static inline int zone_to_nid(struct zone *zone)
  586. {
  587. #ifdef CONFIG_NUMA
  588. return zone->node;
  589. #else
  590. return 0;
  591. #endif
  592. }
  593. #ifdef NODE_NOT_IN_PAGE_FLAGS
  594. extern int page_to_nid(const struct page *page);
  595. #else
  596. static inline int page_to_nid(const struct page *page)
  597. {
  598. return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
  599. }
  600. #endif
  601. #ifdef CONFIG_NUMA_BALANCING
  602. static inline int page_xchg_last_nid(struct page *page, int nid)
  603. {
  604. return xchg(&page->_last_nid, nid);
  605. }
  606. static inline int page_last_nid(struct page *page)
  607. {
  608. return page->_last_nid;
  609. }
  610. static inline void reset_page_last_nid(struct page *page)
  611. {
  612. page->_last_nid = -1;
  613. }
  614. #else
  615. static inline int page_xchg_last_nid(struct page *page, int nid)
  616. {
  617. return page_to_nid(page);
  618. }
  619. static inline int page_last_nid(struct page *page)
  620. {
  621. return page_to_nid(page);
  622. }
  623. static inline void reset_page_last_nid(struct page *page)
  624. {
  625. }
  626. #endif
  627. static inline struct zone *page_zone(const struct page *page)
  628. {
  629. return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
  630. }
  631. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  632. static inline void set_page_section(struct page *page, unsigned long section)
  633. {
  634. page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
  635. page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
  636. }
  637. static inline unsigned long page_to_section(const struct page *page)
  638. {
  639. return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
  640. }
  641. #endif
  642. static inline void set_page_zone(struct page *page, enum zone_type zone)
  643. {
  644. page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
  645. page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
  646. }
  647. static inline void set_page_node(struct page *page, unsigned long node)
  648. {
  649. page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
  650. page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
  651. }
  652. static inline void set_page_links(struct page *page, enum zone_type zone,
  653. unsigned long node, unsigned long pfn)
  654. {
  655. set_page_zone(page, zone);
  656. set_page_node(page, node);
  657. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  658. set_page_section(page, pfn_to_section_nr(pfn));
  659. #endif
  660. }
  661. /*
  662. * Some inline functions in vmstat.h depend on page_zone()
  663. */
  664. #include <linux/vmstat.h>
  665. static __always_inline void *lowmem_page_address(const struct page *page)
  666. {
  667. return __va(PFN_PHYS(page_to_pfn(page)));
  668. }
  669. #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
  670. #define HASHED_PAGE_VIRTUAL
  671. #endif
  672. #if defined(WANT_PAGE_VIRTUAL)
  673. #define page_address(page) ((page)->virtual)
  674. #define set_page_address(page, address) \
  675. do { \
  676. (page)->virtual = (address); \
  677. } while(0)
  678. #define page_address_init() do { } while(0)
  679. #endif
  680. #if defined(HASHED_PAGE_VIRTUAL)
  681. void *page_address(const struct page *page);
  682. void set_page_address(struct page *page, void *virtual);
  683. void page_address_init(void);
  684. #endif
  685. #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
  686. #define page_address(page) lowmem_page_address(page)
  687. #define set_page_address(page, address) do { } while(0)
  688. #define page_address_init() do { } while(0)
  689. #endif
  690. /*
  691. * On an anonymous page mapped into a user virtual memory area,
  692. * page->mapping points to its anon_vma, not to a struct address_space;
  693. * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
  694. *
  695. * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
  696. * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
  697. * and then page->mapping points, not to an anon_vma, but to a private
  698. * structure which KSM associates with that merged page. See ksm.h.
  699. *
  700. * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
  701. *
  702. * Please note that, confusingly, "page_mapping" refers to the inode
  703. * address_space which maps the page from disk; whereas "page_mapped"
  704. * refers to user virtual address space into which the page is mapped.
  705. */
  706. #define PAGE_MAPPING_ANON 1
  707. #define PAGE_MAPPING_KSM 2
  708. #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
  709. extern struct address_space swapper_space;
  710. static inline struct address_space *page_mapping(struct page *page)
  711. {
  712. struct address_space *mapping = page->mapping;
  713. VM_BUG_ON(PageSlab(page));
  714. if (unlikely(PageSwapCache(page)))
  715. mapping = &swapper_space;
  716. else if ((unsigned long)mapping & PAGE_MAPPING_ANON)
  717. mapping = NULL;
  718. return mapping;
  719. }
  720. /* Neutral page->mapping pointer to address_space or anon_vma or other */
  721. static inline void *page_rmapping(struct page *page)
  722. {
  723. return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);
  724. }
  725. extern struct address_space *__page_file_mapping(struct page *);
  726. static inline
  727. struct address_space *page_file_mapping(struct page *page)
  728. {
  729. if (unlikely(PageSwapCache(page)))
  730. return __page_file_mapping(page);
  731. return page->mapping;
  732. }
  733. static inline int PageAnon(struct page *page)
  734. {
  735. return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
  736. }
  737. /*
  738. * Return the pagecache index of the passed page. Regular pagecache pages
  739. * use ->index whereas swapcache pages use ->private
  740. */
  741. static inline pgoff_t page_index(struct page *page)
  742. {
  743. if (unlikely(PageSwapCache(page)))
  744. return page_private(page);
  745. return page->index;
  746. }
  747. extern pgoff_t __page_file_index(struct page *page);
  748. /*
  749. * Return the file index of the page. Regular pagecache pages use ->index
  750. * whereas swapcache pages use swp_offset(->private)
  751. */
  752. static inline pgoff_t page_file_index(struct page *page)
  753. {
  754. if (unlikely(PageSwapCache(page)))
  755. return __page_file_index(page);
  756. return page->index;
  757. }
  758. /*
  759. * Return true if this page is mapped into pagetables.
  760. */
  761. static inline int page_mapped(struct page *page)
  762. {
  763. return atomic_read(&(page)->_mapcount) >= 0;
  764. }
  765. /*
  766. * Different kinds of faults, as returned by handle_mm_fault().
  767. * Used to decide whether a process gets delivered SIGBUS or
  768. * just gets major/minor fault counters bumped up.
  769. */
  770. #define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
  771. #define VM_FAULT_OOM 0x0001
  772. #define VM_FAULT_SIGBUS 0x0002
  773. #define VM_FAULT_MAJOR 0x0004
  774. #define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
  775. #define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
  776. #define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
  777. #define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
  778. #define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
  779. #define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
  780. #define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
  781. #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \
  782. VM_FAULT_HWPOISON_LARGE)
  783. /* Encode hstate index for a hwpoisoned large page */
  784. #define VM_FAULT_SET_HINDEX(x) ((x) << 12)
  785. #define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
  786. /*
  787. * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
  788. */
  789. extern void pagefault_out_of_memory(void);
  790. #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
  791. /*
  792. * Flags passed to show_mem() and show_free_areas() to suppress output in
  793. * various contexts.
  794. */
  795. #define SHOW_MEM_FILTER_NODES (0x0001u) /* filter disallowed nodes */
  796. extern void show_free_areas(unsigned int flags);
  797. extern bool skip_free_areas_node(unsigned int flags, int nid);
  798. int shmem_zero_setup(struct vm_area_struct *);
  799. extern int can_do_mlock(void);
  800. extern int user_shm_lock(size_t, struct user_struct *);
  801. extern void user_shm_unlock(size_t, struct user_struct *);
  802. /*
  803. * Parameter block passed down to zap_pte_range in exceptional cases.
  804. */
  805. struct zap_details {
  806. struct vm_area_struct *nonlinear_vma; /* Check page->index if set */
  807. struct address_space *check_mapping; /* Check page->mapping if set */
  808. pgoff_t first_index; /* Lowest page->index to unmap */
  809. pgoff_t last_index; /* Highest page->index to unmap */
  810. };
  811. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  812. pte_t pte);
  813. int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  814. unsigned long size);
  815. void zap_page_range(struct vm_area_struct *vma, unsigned long address,
  816. unsigned long size, struct zap_details *);
  817. void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
  818. unsigned long start, unsigned long end);
  819. /**
  820. * mm_walk - callbacks for walk_page_range
  821. * @pgd_entry: if set, called for each non-empty PGD (top-level) entry
  822. * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry
  823. * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
  824. * this handler is required to be able to handle
  825. * pmd_trans_huge() pmds. They may simply choose to
  826. * split_huge_page() instead of handling it explicitly.
  827. * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
  828. * @pte_hole: if set, called for each hole at all levels
  829. * @hugetlb_entry: if set, called for each hugetlb entry
  830. * *Caution*: The caller must hold mmap_sem() if @hugetlb_entry
  831. * is used.
  832. *
  833. * (see walk_page_range for more details)
  834. */
  835. struct mm_walk {
  836. int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
  837. int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
  838. int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
  839. int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
  840. int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
  841. int (*hugetlb_entry)(pte_t *, unsigned long,
  842. unsigned long, unsigned long, struct mm_walk *);
  843. struct mm_struct *mm;
  844. void *private;
  845. };
  846. int walk_page_range(unsigned long addr, unsigned long end,
  847. struct mm_walk *walk);
  848. void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
  849. unsigned long end, unsigned long floor, unsigned long ceiling);
  850. int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
  851. struct vm_area_struct *vma);
  852. void unmap_mapping_range(struct address_space *mapping,
  853. loff_t const holebegin, loff_t const holelen, int even_cows);
  854. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  855. unsigned long *pfn);
  856. int follow_phys(struct vm_area_struct *vma, unsigned long address,
  857. unsigned int flags, unsigned long *prot, resource_size_t *phys);
  858. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  859. void *buf, int len, int write);
  860. static inline void unmap_shared_mapping_range(struct address_space *mapping,
  861. loff_t const holebegin, loff_t const holelen)
  862. {
  863. unmap_mapping_range(mapping, holebegin, holelen, 0);
  864. }
  865. extern void truncate_pagecache(struct inode *inode, loff_t old, loff_t new);
  866. extern void truncate_setsize(struct inode *inode, loff_t newsize);
  867. void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
  868. int truncate_inode_page(struct address_space *mapping, struct page *page);
  869. int generic_error_remove_page(struct address_space *mapping, struct page *page);
  870. int invalidate_inode_page(struct page *page);
  871. #ifdef CONFIG_MMU
  872. extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  873. unsigned long address, unsigned int flags);
  874. extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
  875. unsigned long address, unsigned int fault_flags);
  876. #else
  877. static inline int handle_mm_fault(struct mm_struct *mm,
  878. struct vm_area_struct *vma, unsigned long address,
  879. unsigned int flags)
  880. {
  881. /* should never happen if there's no MMU */
  882. BUG();
  883. return VM_FAULT_SIGBUS;
  884. }
  885. static inline int fixup_user_fault(struct task_struct *tsk,
  886. struct mm_struct *mm, unsigned long address,
  887. unsigned int fault_flags)
  888. {
  889. /* should never happen if there's no MMU */
  890. BUG();
  891. return -EFAULT;
  892. }
  893. #endif
  894. extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
  895. extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  896. void *buf, int len, int write);
  897. int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  898. unsigned long start, int len, unsigned int foll_flags,
  899. struct page **pages, struct vm_area_struct **vmas,
  900. int *nonblocking);
  901. int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  902. unsigned long start, int nr_pages, int write, int force,
  903. struct page **pages, struct vm_area_struct **vmas);
  904. int get_user_pages_fast(unsigned long start, int nr_pages, int write,
  905. struct page **pages);
  906. struct kvec;
  907. int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
  908. struct page **pages);
  909. int get_kernel_page(unsigned long start, int write, struct page **pages);
  910. struct page *get_dump_page(unsigned long addr);
  911. extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
  912. extern void do_invalidatepage(struct page *page, unsigned long offset);
  913. int __set_page_dirty_nobuffers(struct page *page);
  914. int __set_page_dirty_no_writeback(struct page *page);
  915. int redirty_page_for_writepage(struct writeback_control *wbc,
  916. struct page *page);
  917. void account_page_dirtied(struct page *page, struct address_space *mapping);
  918. void account_page_writeback(struct page *page);
  919. int set_page_dirty(struct page *page);
  920. int set_page_dirty_lock(struct page *page);
  921. int clear_page_dirty_for_io(struct page *page);
  922. /* Is the vma a continuation of the stack vma above it? */
  923. static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
  924. {
  925. return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
  926. }
  927. static inline int stack_guard_page_start(struct vm_area_struct *vma,
  928. unsigned long addr)
  929. {
  930. return (vma->vm_flags & VM_GROWSDOWN) &&
  931. (vma->vm_start == addr) &&
  932. !vma_growsdown(vma->vm_prev, addr);
  933. }
  934. /* Is the vma a continuation of the stack vma below it? */
  935. static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
  936. {
  937. return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
  938. }
  939. static inline int stack_guard_page_end(struct vm_area_struct *vma,
  940. unsigned long addr)
  941. {
  942. return (vma->vm_flags & VM_GROWSUP) &&
  943. (vma->vm_end == addr) &&
  944. !vma_growsup(vma->vm_next, addr);
  945. }
  946. extern pid_t
  947. vm_is_stack(struct task_struct *task, struct vm_area_struct *vma, int in_group);
  948. extern unsigned long move_page_tables(struct vm_area_struct *vma,
  949. unsigned long old_addr, struct vm_area_struct *new_vma,
  950. unsigned long new_addr, unsigned long len,
  951. bool need_rmap_locks);
  952. extern unsigned long do_mremap(unsigned long addr,
  953. unsigned long old_len, unsigned long new_len,
  954. unsigned long flags, unsigned long new_addr);
  955. extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
  956. unsigned long end, pgprot_t newprot,
  957. int dirty_accountable, int prot_numa);
  958. extern int mprotect_fixup(struct vm_area_struct *vma,
  959. struct vm_area_struct **pprev, unsigned long start,
  960. unsigned long end, unsigned long newflags);
  961. /*
  962. * doesn't attempt to fault and will return short.
  963. */
  964. int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
  965. struct page **pages);
  966. /*
  967. * per-process(per-mm_struct) statistics.
  968. */
  969. static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
  970. {
  971. long val = atomic_long_read(&mm->rss_stat.count[member]);
  972. #ifdef SPLIT_RSS_COUNTING
  973. /*
  974. * counter is updated in asynchronous manner and may go to minus.
  975. * But it's never be expected number for users.
  976. */
  977. if (val < 0)
  978. val = 0;
  979. #endif
  980. return (unsigned long)val;
  981. }
  982. static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
  983. {
  984. atomic_long_add(value, &mm->rss_stat.count[member]);
  985. }
  986. static inline void inc_mm_counter(struct mm_struct *mm, int member)
  987. {
  988. atomic_long_inc(&mm->rss_stat.count[member]);
  989. }
  990. static inline void dec_mm_counter(struct mm_struct *mm, int member)
  991. {
  992. atomic_long_dec(&mm->rss_stat.count[member]);
  993. }
  994. static inline unsigned long get_mm_rss(struct mm_struct *mm)
  995. {
  996. return get_mm_counter(mm, MM_FILEPAGES) +
  997. get_mm_counter(mm, MM_ANONPAGES);
  998. }
  999. static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
  1000. {
  1001. return max(mm->hiwater_rss, get_mm_rss(mm));
  1002. }
  1003. static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
  1004. {
  1005. return max(mm->hiwater_vm, mm->total_vm);
  1006. }
  1007. static inline void update_hiwater_rss(struct mm_struct *mm)
  1008. {
  1009. unsigned long _rss = get_mm_rss(mm);
  1010. if ((mm)->hiwater_rss < _rss)
  1011. (mm)->hiwater_rss = _rss;
  1012. }
  1013. static inline void update_hiwater_vm(struct mm_struct *mm)
  1014. {
  1015. if (mm->hiwater_vm < mm->total_vm)
  1016. mm->hiwater_vm = mm->total_vm;
  1017. }
  1018. static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
  1019. struct mm_struct *mm)
  1020. {
  1021. unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
  1022. if (*maxrss < hiwater_rss)
  1023. *maxrss = hiwater_rss;
  1024. }
  1025. #if defined(SPLIT_RSS_COUNTING)
  1026. void sync_mm_rss(struct mm_struct *mm);
  1027. #else
  1028. static inline void sync_mm_rss(struct mm_struct *mm)
  1029. {
  1030. }
  1031. #endif
  1032. int vma_wants_writenotify(struct vm_area_struct *vma);
  1033. extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1034. spinlock_t **ptl);
  1035. static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1036. spinlock_t **ptl)
  1037. {
  1038. pte_t *ptep;
  1039. __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
  1040. return ptep;
  1041. }
  1042. #ifdef __PAGETABLE_PUD_FOLDED
  1043. static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
  1044. unsigned long address)
  1045. {
  1046. return 0;
  1047. }
  1048. #else
  1049. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
  1050. #endif
  1051. #ifdef __PAGETABLE_PMD_FOLDED
  1052. static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
  1053. unsigned long address)
  1054. {
  1055. return 0;
  1056. }
  1057. #else
  1058. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
  1059. #endif
  1060. int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
  1061. pmd_t *pmd, unsigned long address);
  1062. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
  1063. /*
  1064. * The following ifdef needed to get the 4level-fixup.h header to work.
  1065. * Remove it when 4level-fixup.h has been removed.
  1066. */
  1067. #if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
  1068. static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  1069. {
  1070. return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
  1071. NULL: pud_offset(pgd, address);
  1072. }
  1073. static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  1074. {
  1075. return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
  1076. NULL: pmd_offset(pud, address);
  1077. }
  1078. #endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
  1079. #if USE_SPLIT_PTLOCKS
  1080. /*
  1081. * We tuck a spinlock to guard each pagetable page into its struct page,
  1082. * at page->private, with BUILD_BUG_ON to make sure that this will not
  1083. * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
  1084. * When freeing, reset page->mapping so free_pages_check won't complain.
  1085. */
  1086. #define __pte_lockptr(page) &((page)->ptl)
  1087. #define pte_lock_init(_page) do { \
  1088. spin_lock_init(__pte_lockptr(_page)); \
  1089. } while (0)
  1090. #define pte_lock_deinit(page) ((page)->mapping = NULL)
  1091. #define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
  1092. #else /* !USE_SPLIT_PTLOCKS */
  1093. /*
  1094. * We use mm->page_table_lock to guard all pagetable pages of the mm.
  1095. */
  1096. #define pte_lock_init(page) do {} while (0)
  1097. #define pte_lock_deinit(page) do {} while (0)
  1098. #define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
  1099. #endif /* USE_SPLIT_PTLOCKS */
  1100. static inline void pgtable_page_ctor(struct page *page)
  1101. {
  1102. pte_lock_init(page);
  1103. inc_zone_page_state(page, NR_PAGETABLE);
  1104. }
  1105. static inline void pgtable_page_dtor(struct page *page)
  1106. {
  1107. pte_lock_deinit(page);
  1108. dec_zone_page_state(page, NR_PAGETABLE);
  1109. }
  1110. #define pte_offset_map_lock(mm, pmd, address, ptlp) \
  1111. ({ \
  1112. spinlock_t *__ptl = pte_lockptr(mm, pmd); \
  1113. pte_t *__pte = pte_offset_map(pmd, address); \
  1114. *(ptlp) = __ptl; \
  1115. spin_lock(__ptl); \
  1116. __pte; \
  1117. })
  1118. #define pte_unmap_unlock(pte, ptl) do { \
  1119. spin_unlock(ptl); \
  1120. pte_unmap(pte); \
  1121. } while (0)
  1122. #define pte_alloc_map(mm, vma, pmd, address) \
  1123. ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
  1124. pmd, address))? \
  1125. NULL: pte_offset_map(pmd, address))
  1126. #define pte_alloc_map_lock(mm, pmd, address, ptlp) \
  1127. ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
  1128. pmd, address))? \
  1129. NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
  1130. #define pte_alloc_kernel(pmd, address) \
  1131. ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
  1132. NULL: pte_offset_kernel(pmd, address))
  1133. extern void free_area_init(unsigned long * zones_size);
  1134. extern void free_area_init_node(int nid, unsigned long * zones_size,
  1135. unsigned long zone_start_pfn, unsigned long *zholes_size);
  1136. extern void free_initmem(void);
  1137. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  1138. /*
  1139. * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
  1140. * zones, allocate the backing mem_map and account for memory holes in a more
  1141. * architecture independent manner. This is a substitute for creating the
  1142. * zone_sizes[] and zholes_size[] arrays and passing them to
  1143. * free_area_init_node()
  1144. *
  1145. * An architecture is expected to register range of page frames backed by
  1146. * physical memory with memblock_add[_node]() before calling
  1147. * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
  1148. * usage, an architecture is expected to do something like
  1149. *
  1150. * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
  1151. * max_highmem_pfn};
  1152. * for_each_valid_physical_page_range()
  1153. * memblock_add_node(base, size, nid)
  1154. * free_area_init_nodes(max_zone_pfns);
  1155. *
  1156. * free_bootmem_with_active_regions() calls free_bootmem_node() for each
  1157. * registered physical page range. Similarly
  1158. * sparse_memory_present_with_active_regions() calls memory_present() for
  1159. * each range when SPARSEMEM is enabled.
  1160. *
  1161. * See mm/page_alloc.c for more information on each function exposed by
  1162. * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
  1163. */
  1164. extern void free_area_init_nodes(unsigned long *max_zone_pfn);
  1165. unsigned long node_map_pfn_alignment(void);
  1166. unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
  1167. unsigned long end_pfn);
  1168. extern unsigned long absent_pages_in_range(unsigned long start_pfn,
  1169. unsigned long end_pfn);
  1170. extern void get_pfn_range_for_nid(unsigned int nid,
  1171. unsigned long *start_pfn, unsigned long *end_pfn);
  1172. extern unsigned long find_min_pfn_with_active_regions(void);
  1173. extern void free_bootmem_with_active_regions(int nid,
  1174. unsigned long max_low_pfn);
  1175. extern void sparse_memory_present_with_active_regions(int nid);
  1176. #define MOVABLEMEM_MAP_MAX MAX_NUMNODES
  1177. struct movablemem_entry {
  1178. unsigned long start_pfn; /* start pfn of memory segment */
  1179. unsigned long end_pfn; /* end pfn of memory segment (exclusive) */
  1180. };
  1181. struct movablemem_map {
  1182. int nr_map;
  1183. struct movablemem_entry map[MOVABLEMEM_MAP_MAX];
  1184. };
  1185. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  1186. #if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
  1187. !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
  1188. static inline int __early_pfn_to_nid(unsigned long pfn)
  1189. {
  1190. return 0;
  1191. }
  1192. #else
  1193. /* please see mm/page_alloc.c */
  1194. extern int __meminit early_pfn_to_nid(unsigned long pfn);
  1195. #ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  1196. /* there is a per-arch backend function. */
  1197. extern int __meminit __early_pfn_to_nid(unsigned long pfn);
  1198. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  1199. #endif
  1200. extern void set_dma_reserve(unsigned long new_dma_reserve);
  1201. extern void memmap_init_zone(unsigned long, int, unsigned long,
  1202. unsigned long, enum memmap_context);
  1203. extern void setup_per_zone_wmarks(void);
  1204. extern int __meminit init_per_zone_wmark_min(void);
  1205. extern void mem_init(void);
  1206. extern void __init mmap_init(void);
  1207. extern void show_mem(unsigned int flags);
  1208. extern void si_meminfo(struct sysinfo * val);
  1209. extern void si_meminfo_node(struct sysinfo *val, int nid);
  1210. extern __printf(3, 4)
  1211. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
  1212. extern void setup_per_cpu_pageset(void);
  1213. extern void zone_pcp_update(struct zone *zone);
  1214. extern void zone_pcp_reset(struct zone *zone);
  1215. /* nommu.c */
  1216. extern atomic_long_t mmap_pages_allocated;
  1217. extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
  1218. /* interval_tree.c */
  1219. void vma_interval_tree_insert(struct vm_area_struct *node,
  1220. struct rb_root *root);
  1221. void vma_interval_tree_insert_after(struct vm_area_struct *node,
  1222. struct vm_area_struct *prev,
  1223. struct rb_root *root);
  1224. void vma_interval_tree_remove(struct vm_area_struct *node,
  1225. struct rb_root *root);
  1226. struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
  1227. unsigned long start, unsigned long last);
  1228. struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
  1229. unsigned long start, unsigned long last);
  1230. #define vma_interval_tree_foreach(vma, root, start, last) \
  1231. for (vma = vma_interval_tree_iter_first(root, start, last); \
  1232. vma; vma = vma_interval_tree_iter_next(vma, start, last))
  1233. static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
  1234. struct list_head *list)
  1235. {
  1236. list_add_tail(&vma->shared.nonlinear, list);
  1237. }
  1238. void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
  1239. struct rb_root *root);
  1240. void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
  1241. struct rb_root *root);
  1242. struct anon_vma_chain *anon_vma_interval_tree_iter_first(
  1243. struct rb_root *root, unsigned long start, unsigned long last);
  1244. struct anon_vma_chain *anon_vma_interval_tree_iter_next(
  1245. struct anon_vma_chain *node, unsigned long start, unsigned long last);
  1246. #ifdef CONFIG_DEBUG_VM_RB
  1247. void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
  1248. #endif
  1249. #define anon_vma_interval_tree_foreach(avc, root, start, last) \
  1250. for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
  1251. avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
  1252. /* mmap.c */
  1253. extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
  1254. extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
  1255. unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
  1256. extern struct vm_area_struct *vma_merge(struct mm_struct *,
  1257. struct vm_area_struct *prev, unsigned long addr, unsigned long end,
  1258. unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
  1259. struct mempolicy *);
  1260. extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
  1261. extern int split_vma(struct mm_struct *,
  1262. struct vm_area_struct *, unsigned long addr, int new_below);
  1263. extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
  1264. extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
  1265. struct rb_node **, struct rb_node *);
  1266. extern void unlink_file_vma(struct vm_area_struct *);
  1267. extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
  1268. unsigned long addr, unsigned long len, pgoff_t pgoff,
  1269. bool *need_rmap_locks);
  1270. extern void exit_mmap(struct mm_struct *);
  1271. extern int mm_take_all_locks(struct mm_struct *mm);
  1272. extern void mm_drop_all_locks(struct mm_struct *mm);
  1273. extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
  1274. extern struct file *get_mm_exe_file(struct mm_struct *mm);
  1275. extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
  1276. extern int install_special_mapping(struct mm_struct *mm,
  1277. unsigned long addr, unsigned long len,
  1278. unsigned long flags, struct page **pages);
  1279. extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
  1280. extern unsigned long mmap_region(struct file *file, unsigned long addr,
  1281. unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
  1282. extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
  1283. unsigned long len, unsigned long prot, unsigned long flags,
  1284. unsigned long pgoff, unsigned long *populate);
  1285. extern int do_munmap(struct mm_struct *, unsigned long, size_t);
  1286. #ifdef CONFIG_MMU
  1287. extern int __mm_populate(unsigned long addr, unsigned long len,
  1288. int ignore_errors);
  1289. static inline void mm_populate(unsigned long addr, unsigned long len)
  1290. {
  1291. /* Ignore errors */
  1292. (void) __mm_populate(addr, len, 1);
  1293. }
  1294. #else
  1295. static inline void mm_populate(unsigned long addr, unsigned long len) {}
  1296. #endif
  1297. /* These take the mm semaphore themselves */
  1298. extern unsigned long vm_brk(unsigned long, unsigned long);
  1299. extern int vm_munmap(unsigned long, size_t);
  1300. extern unsigned long vm_mmap(struct file *, unsigned long,
  1301. unsigned long, unsigned long,
  1302. unsigned long, unsigned long);
  1303. struct vm_unmapped_area_info {
  1304. #define VM_UNMAPPED_AREA_TOPDOWN 1
  1305. unsigned long flags;
  1306. unsigned long length;
  1307. unsigned long low_limit;
  1308. unsigned long high_limit;
  1309. unsigned long align_mask;
  1310. unsigned long align_offset;
  1311. };
  1312. extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
  1313. extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
  1314. /*
  1315. * Search for an unmapped address range.
  1316. *
  1317. * We are looking for a range that:
  1318. * - does not intersect with any VMA;
  1319. * - is contained within the [low_limit, high_limit) interval;
  1320. * - is at least the desired size.
  1321. * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
  1322. */
  1323. static inline unsigned long
  1324. vm_unmapped_area(struct vm_unmapped_area_info *info)
  1325. {
  1326. if (!(info->flags & VM_UNMAPPED_AREA_TOPDOWN))
  1327. return unmapped_area(info);
  1328. else
  1329. return unmapped_area_topdown(info);
  1330. }
  1331. /* truncate.c */
  1332. extern void truncate_inode_pages(struct address_space *, loff_t);
  1333. extern void truncate_inode_pages_range(struct address_space *,
  1334. loff_t lstart, loff_t lend);
  1335. /* generic vm_area_ops exported for stackable file systems */
  1336. extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
  1337. extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
  1338. /* mm/page-writeback.c */
  1339. int write_one_page(struct page *page, int wait);
  1340. void task_dirty_inc(struct task_struct *tsk);
  1341. /* readahead.c */
  1342. #define VM_MAX_READAHEAD 128 /* kbytes */
  1343. #define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
  1344. int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
  1345. pgoff_t offset, unsigned long nr_to_read);
  1346. void page_cache_sync_readahead(struct address_space *mapping,
  1347. struct file_ra_state *ra,
  1348. struct file *filp,
  1349. pgoff_t offset,
  1350. unsigned long size);
  1351. void page_cache_async_readahead(struct address_space *mapping,
  1352. struct file_ra_state *ra,
  1353. struct file *filp,
  1354. struct page *pg,
  1355. pgoff_t offset,
  1356. unsigned long size);
  1357. unsigned long max_sane_readahead(unsigned long nr);
  1358. unsigned long ra_submit(struct file_ra_state *ra,
  1359. struct address_space *mapping,
  1360. struct file *filp);
  1361. /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
  1362. extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
  1363. /* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
  1364. extern int expand_downwards(struct vm_area_struct *vma,
  1365. unsigned long address);
  1366. #if VM_GROWSUP
  1367. extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
  1368. #else
  1369. #define expand_upwards(vma, address) do { } while (0)
  1370. #endif
  1371. /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
  1372. extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
  1373. extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
  1374. struct vm_area_struct **pprev);
  1375. /* Look up the first VMA which intersects the interval start_addr..end_addr-1,
  1376. NULL if none. Assume start_addr < end_addr. */
  1377. static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
  1378. {
  1379. struct vm_area_struct * vma = find_vma(mm,start_addr);
  1380. if (vma && end_addr <= vma->vm_start)
  1381. vma = NULL;
  1382. return vma;
  1383. }
  1384. static inline unsigned long vma_pages(struct vm_area_struct *vma)
  1385. {
  1386. return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  1387. }
  1388. /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
  1389. static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
  1390. unsigned long vm_start, unsigned long vm_end)
  1391. {
  1392. struct vm_area_struct *vma = find_vma(mm, vm_start);
  1393. if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
  1394. vma = NULL;
  1395. return vma;
  1396. }
  1397. #ifdef CONFIG_MMU
  1398. pgprot_t vm_get_page_prot(unsigned long vm_flags);
  1399. #else
  1400. static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
  1401. {
  1402. return __pgprot(0);
  1403. }
  1404. #endif
  1405. #ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
  1406. unsigned long change_prot_numa(struct vm_area_struct *vma,
  1407. unsigned long start, unsigned long end);
  1408. #endif
  1409. struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
  1410. int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
  1411. unsigned long pfn, unsigned long size, pgprot_t);
  1412. int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
  1413. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1414. unsigned long pfn);
  1415. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1416. unsigned long pfn);
  1417. struct page *follow_page(struct vm_area_struct *, unsigned long address,
  1418. unsigned int foll_flags);
  1419. #define FOLL_WRITE 0x01 /* check pte is writable */
  1420. #define FOLL_TOUCH 0x02 /* mark page accessed */
  1421. #define FOLL_GET 0x04 /* do get_page on page */
  1422. #define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
  1423. #define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
  1424. #define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
  1425. * and return without waiting upon it */
  1426. #define FOLL_MLOCK 0x40 /* mark page as mlocked */
  1427. #define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
  1428. #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
  1429. #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
  1430. typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
  1431. void *data);
  1432. extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
  1433. unsigned long size, pte_fn_t fn, void *data);
  1434. #ifdef CONFIG_PROC_FS
  1435. void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
  1436. #else
  1437. static inline void vm_stat_account(struct mm_struct *mm,
  1438. unsigned long flags, struct file *file, long pages)
  1439. {
  1440. mm->total_vm += pages;
  1441. }
  1442. #endif /* CONFIG_PROC_FS */
  1443. #ifdef CONFIG_DEBUG_PAGEALLOC
  1444. extern void kernel_map_pages(struct page *page, int numpages, int enable);
  1445. #ifdef CONFIG_HIBERNATION
  1446. extern bool kernel_page_present(struct page *page);
  1447. #endif /* CONFIG_HIBERNATION */
  1448. #else
  1449. static inline void
  1450. kernel_map_pages(struct page *page, int numpages, int enable) {}
  1451. #ifdef CONFIG_HIBERNATION
  1452. static inline bool kernel_page_present(struct page *page) { return true; }
  1453. #endif /* CONFIG_HIBERNATION */
  1454. #endif
  1455. extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
  1456. #ifdef __HAVE_ARCH_GATE_AREA
  1457. int in_gate_area_no_mm(unsigned long addr);
  1458. int in_gate_area(struct mm_struct *mm, unsigned long addr);
  1459. #else
  1460. int in_gate_area_no_mm(unsigned long addr);
  1461. #define in_gate_area(mm, addr) ({(void)mm; in_gate_area_no_mm(addr);})
  1462. #endif /* __HAVE_ARCH_GATE_AREA */
  1463. int drop_caches_sysctl_handler(struct ctl_table *, int,
  1464. void __user *, size_t *, loff_t *);
  1465. unsigned long shrink_slab(struct shrink_control *shrink,
  1466. unsigned long nr_pages_scanned,
  1467. unsigned long lru_pages);
  1468. #ifndef CONFIG_MMU
  1469. #define randomize_va_space 0
  1470. #else
  1471. extern int randomize_va_space;
  1472. #endif
  1473. const char * arch_vma_name(struct vm_area_struct *vma);
  1474. void print_vma_addr(char *prefix, unsigned long rip);
  1475. void sparse_mem_maps_populate_node(struct page **map_map,
  1476. unsigned long pnum_begin,
  1477. unsigned long pnum_end,
  1478. unsigned long map_count,
  1479. int nodeid);
  1480. struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
  1481. pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
  1482. pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
  1483. pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
  1484. pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
  1485. void *vmemmap_alloc_block(unsigned long size, int node);
  1486. void *vmemmap_alloc_block_buf(unsigned long size, int node);
  1487. void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
  1488. int vmemmap_populate_basepages(struct page *start_page,
  1489. unsigned long pages, int node);
  1490. int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
  1491. void vmemmap_populate_print_last(void);
  1492. #ifdef CONFIG_MEMORY_HOTPLUG
  1493. void vmemmap_free(struct page *memmap, unsigned long nr_pages);
  1494. #endif
  1495. void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
  1496. unsigned long size);
  1497. enum mf_flags {
  1498. MF_COUNT_INCREASED = 1 << 0,
  1499. MF_ACTION_REQUIRED = 1 << 1,
  1500. MF_MUST_KILL = 1 << 2,
  1501. };
  1502. extern int memory_failure(unsigned long pfn, int trapno, int flags);
  1503. extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
  1504. extern int unpoison_memory(unsigned long pfn);
  1505. extern int sysctl_memory_failure_early_kill;
  1506. extern int sysctl_memory_failure_recovery;
  1507. extern void shake_page(struct page *p, int access);
  1508. extern atomic_long_t mce_bad_pages;
  1509. extern int soft_offline_page(struct page *page, int flags);
  1510. extern void dump_page(struct page *page);
  1511. #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
  1512. extern void clear_huge_page(struct page *page,
  1513. unsigned long addr,
  1514. unsigned int pages_per_huge_page);
  1515. extern void copy_user_huge_page(struct page *dst, struct page *src,
  1516. unsigned long addr, struct vm_area_struct *vma,
  1517. unsigned int pages_per_huge_page);
  1518. #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
  1519. #ifdef CONFIG_DEBUG_PAGEALLOC
  1520. extern unsigned int _debug_guardpage_minorder;
  1521. static inline unsigned int debug_guardpage_minorder(void)
  1522. {
  1523. return _debug_guardpage_minorder;
  1524. }
  1525. static inline bool page_is_guard(struct page *page)
  1526. {
  1527. return test_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  1528. }
  1529. #else
  1530. static inline unsigned int debug_guardpage_minorder(void) { return 0; }
  1531. static inline bool page_is_guard(struct page *page) { return false; }
  1532. #endif /* CONFIG_DEBUG_PAGEALLOC */
  1533. #endif /* __KERNEL__ */
  1534. #endif /* _LINUX_MM_H */