mm.h 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886
  1. #ifndef _LINUX_MM_H
  2. #define _LINUX_MM_H
  3. #include <linux/errno.h>
  4. #ifdef __KERNEL__
  5. #include <linux/gfp.h>
  6. #include <linux/bug.h>
  7. #include <linux/list.h>
  8. #include <linux/mmzone.h>
  9. #include <linux/rbtree.h>
  10. #include <linux/atomic.h>
  11. #include <linux/debug_locks.h>
  12. #include <linux/mm_types.h>
  13. #include <linux/range.h>
  14. #include <linux/pfn.h>
  15. #include <linux/bit_spinlock.h>
  16. #include <linux/shrinker.h>
  17. struct mempolicy;
  18. struct anon_vma;
  19. struct anon_vma_chain;
  20. struct file_ra_state;
  21. struct user_struct;
  22. struct writeback_control;
  23. #ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
  24. extern unsigned long max_mapnr;
  25. static inline void set_max_mapnr(unsigned long limit)
  26. {
  27. max_mapnr = limit;
  28. }
  29. #else
  30. static inline void set_max_mapnr(unsigned long limit) { }
  31. #endif
  32. extern unsigned long totalram_pages;
  33. extern void * high_memory;
  34. extern int page_cluster;
  35. #ifdef CONFIG_SYSCTL
  36. extern int sysctl_legacy_va_layout;
  37. #else
  38. #define sysctl_legacy_va_layout 0
  39. #endif
  40. #include <asm/page.h>
  41. #include <asm/pgtable.h>
  42. #include <asm/processor.h>
  43. extern unsigned long sysctl_user_reserve_kbytes;
  44. extern unsigned long sysctl_admin_reserve_kbytes;
  45. #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
  46. /* to align the pointer to the (next) page boundary */
  47. #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
  48. /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
  49. #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
  50. /*
  51. * Linux kernel virtual memory manager primitives.
  52. * The idea being to have a "virtual" mm in the same way
  53. * we have a virtual fs - giving a cleaner interface to the
  54. * mm details, and allowing different kinds of memory mappings
  55. * (from shared memory to executable loading to arbitrary
  56. * mmap() functions).
  57. */
  58. extern struct kmem_cache *vm_area_cachep;
  59. #ifndef CONFIG_MMU
  60. extern struct rb_root nommu_region_tree;
  61. extern struct rw_semaphore nommu_region_sem;
  62. extern unsigned int kobjsize(const void *objp);
  63. #endif
  64. /*
  65. * vm_flags in vm_area_struct, see mm_types.h.
  66. */
  67. #define VM_NONE 0x00000000
  68. #define VM_READ 0x00000001 /* currently active flags */
  69. #define VM_WRITE 0x00000002
  70. #define VM_EXEC 0x00000004
  71. #define VM_SHARED 0x00000008
  72. /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
  73. #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
  74. #define VM_MAYWRITE 0x00000020
  75. #define VM_MAYEXEC 0x00000040
  76. #define VM_MAYSHARE 0x00000080
  77. #define VM_GROWSDOWN 0x00000100 /* general info on the segment */
  78. #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
  79. #define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
  80. #define VM_LOCKED 0x00002000
  81. #define VM_IO 0x00004000 /* Memory mapped I/O or similar */
  82. /* Used by sys_madvise() */
  83. #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
  84. #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
  85. #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
  86. #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
  87. #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
  88. #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
  89. #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
  90. #define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */
  91. #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
  92. #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
  93. #ifdef CONFIG_MEM_SOFT_DIRTY
  94. # define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
  95. #else
  96. # define VM_SOFTDIRTY 0
  97. #endif
  98. #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
  99. #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
  100. #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
  101. #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
  102. #if defined(CONFIG_X86)
  103. # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
  104. #elif defined(CONFIG_PPC)
  105. # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
  106. #elif defined(CONFIG_PARISC)
  107. # define VM_GROWSUP VM_ARCH_1
  108. #elif defined(CONFIG_METAG)
  109. # define VM_GROWSUP VM_ARCH_1
  110. #elif defined(CONFIG_IA64)
  111. # define VM_GROWSUP VM_ARCH_1
  112. #elif !defined(CONFIG_MMU)
  113. # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
  114. #endif
  115. #ifndef VM_GROWSUP
  116. # define VM_GROWSUP VM_NONE
  117. #endif
  118. /* Bits set in the VMA until the stack is in its final location */
  119. #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
  120. #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
  121. #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
  122. #endif
  123. #ifdef CONFIG_STACK_GROWSUP
  124. #define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  125. #else
  126. #define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  127. #endif
  128. /*
  129. * Special vmas that are non-mergable, non-mlock()able.
  130. * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
  131. */
  132. #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)
  133. /*
  134. * mapping from the currently active vm_flags protection bits (the
  135. * low four bits) to a page protection mask..
  136. */
  137. extern pgprot_t protection_map[16];
  138. #define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
  139. #define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */
  140. #define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */
  141. #define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */
  142. #define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */
  143. #define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */
  144. #define FAULT_FLAG_TRIED 0x40 /* second try */
  145. #define FAULT_FLAG_USER 0x80 /* The fault originated in userspace */
  146. /*
  147. * vm_fault is filled by the the pagefault handler and passed to the vma's
  148. * ->fault function. The vma's ->fault is responsible for returning a bitmask
  149. * of VM_FAULT_xxx flags that give details about how the fault was handled.
  150. *
  151. * pgoff should be used in favour of virtual_address, if possible. If pgoff
  152. * is used, one may implement ->remap_pages to get nonlinear mapping support.
  153. */
  154. struct vm_fault {
  155. unsigned int flags; /* FAULT_FLAG_xxx flags */
  156. pgoff_t pgoff; /* Logical page offset based on vma */
  157. void __user *virtual_address; /* Faulting virtual address */
  158. struct page *page; /* ->fault handlers should return a
  159. * page here, unless VM_FAULT_NOPAGE
  160. * is set (which is also implied by
  161. * VM_FAULT_ERROR).
  162. */
  163. };
  164. /*
  165. * These are the virtual MM functions - opening of an area, closing and
  166. * unmapping it (needed to keep files on disk up-to-date etc), pointer
  167. * to the functions called when a no-page or a wp-page exception occurs.
  168. */
  169. struct vm_operations_struct {
  170. void (*open)(struct vm_area_struct * area);
  171. void (*close)(struct vm_area_struct * area);
  172. int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
  173. /* notification that a previously read-only page is about to become
  174. * writable, if an error is returned it will cause a SIGBUS */
  175. int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
  176. /* called by access_process_vm when get_user_pages() fails, typically
  177. * for use by special VMAs that can switch between memory and hardware
  178. */
  179. int (*access)(struct vm_area_struct *vma, unsigned long addr,
  180. void *buf, int len, int write);
  181. #ifdef CONFIG_NUMA
  182. /*
  183. * set_policy() op must add a reference to any non-NULL @new mempolicy
  184. * to hold the policy upon return. Caller should pass NULL @new to
  185. * remove a policy and fall back to surrounding context--i.e. do not
  186. * install a MPOL_DEFAULT policy, nor the task or system default
  187. * mempolicy.
  188. */
  189. int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
  190. /*
  191. * get_policy() op must add reference [mpol_get()] to any policy at
  192. * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
  193. * in mm/mempolicy.c will do this automatically.
  194. * get_policy() must NOT add a ref if the policy at (vma,addr) is not
  195. * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
  196. * If no [shared/vma] mempolicy exists at the addr, get_policy() op
  197. * must return NULL--i.e., do not "fallback" to task or system default
  198. * policy.
  199. */
  200. struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
  201. unsigned long addr);
  202. int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
  203. const nodemask_t *to, unsigned long flags);
  204. #endif
  205. /* called by sys_remap_file_pages() to populate non-linear mapping */
  206. int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
  207. unsigned long size, pgoff_t pgoff);
  208. };
  209. struct mmu_gather;
  210. struct inode;
  211. #define page_private(page) ((page)->private)
  212. #define set_page_private(page, v) ((page)->private = (v))
  213. /* It's valid only if the page is free path or free_list */
  214. static inline void set_freepage_migratetype(struct page *page, int migratetype)
  215. {
  216. page->index = migratetype;
  217. }
  218. /* It's valid only if the page is free path or free_list */
  219. static inline int get_freepage_migratetype(struct page *page)
  220. {
  221. return page->index;
  222. }
  223. /*
  224. * FIXME: take this include out, include page-flags.h in
  225. * files which need it (119 of them)
  226. */
  227. #include <linux/page-flags.h>
  228. #include <linux/huge_mm.h>
  229. /*
  230. * Methods to modify the page usage count.
  231. *
  232. * What counts for a page usage:
  233. * - cache mapping (page->mapping)
  234. * - private data (page->private)
  235. * - page mapped in a task's page tables, each mapping
  236. * is counted separately
  237. *
  238. * Also, many kernel routines increase the page count before a critical
  239. * routine so they can be sure the page doesn't go away from under them.
  240. */
  241. /*
  242. * Drop a ref, return true if the refcount fell to zero (the page has no users)
  243. */
  244. static inline int put_page_testzero(struct page *page)
  245. {
  246. VM_BUG_ON(atomic_read(&page->_count) == 0);
  247. return atomic_dec_and_test(&page->_count);
  248. }
  249. /*
  250. * Try to grab a ref unless the page has a refcount of zero, return false if
  251. * that is the case.
  252. */
  253. static inline int get_page_unless_zero(struct page *page)
  254. {
  255. return atomic_inc_not_zero(&page->_count);
  256. }
  257. extern int page_is_ram(unsigned long pfn);
  258. /* Support for virtually mapped pages */
  259. struct page *vmalloc_to_page(const void *addr);
  260. unsigned long vmalloc_to_pfn(const void *addr);
  261. /*
  262. * Determine if an address is within the vmalloc range
  263. *
  264. * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
  265. * is no special casing required.
  266. */
  267. static inline int is_vmalloc_addr(const void *x)
  268. {
  269. #ifdef CONFIG_MMU
  270. unsigned long addr = (unsigned long)x;
  271. return addr >= VMALLOC_START && addr < VMALLOC_END;
  272. #else
  273. return 0;
  274. #endif
  275. }
  276. #ifdef CONFIG_MMU
  277. extern int is_vmalloc_or_module_addr(const void *x);
  278. #else
  279. static inline int is_vmalloc_or_module_addr(const void *x)
  280. {
  281. return 0;
  282. }
  283. #endif
  284. static inline void compound_lock(struct page *page)
  285. {
  286. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  287. VM_BUG_ON(PageSlab(page));
  288. bit_spin_lock(PG_compound_lock, &page->flags);
  289. #endif
  290. }
  291. static inline void compound_unlock(struct page *page)
  292. {
  293. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  294. VM_BUG_ON(PageSlab(page));
  295. bit_spin_unlock(PG_compound_lock, &page->flags);
  296. #endif
  297. }
  298. static inline unsigned long compound_lock_irqsave(struct page *page)
  299. {
  300. unsigned long uninitialized_var(flags);
  301. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  302. local_irq_save(flags);
  303. compound_lock(page);
  304. #endif
  305. return flags;
  306. }
  307. static inline void compound_unlock_irqrestore(struct page *page,
  308. unsigned long flags)
  309. {
  310. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  311. compound_unlock(page);
  312. local_irq_restore(flags);
  313. #endif
  314. }
  315. static inline struct page *compound_head(struct page *page)
  316. {
  317. if (unlikely(PageTail(page)))
  318. return page->first_page;
  319. return page;
  320. }
  321. /*
  322. * The atomic page->_mapcount, starts from -1: so that transitions
  323. * both from it and to it can be tracked, using atomic_inc_and_test
  324. * and atomic_add_negative(-1).
  325. */
  326. static inline void page_mapcount_reset(struct page *page)
  327. {
  328. atomic_set(&(page)->_mapcount, -1);
  329. }
  330. static inline int page_mapcount(struct page *page)
  331. {
  332. return atomic_read(&(page)->_mapcount) + 1;
  333. }
  334. static inline int page_count(struct page *page)
  335. {
  336. return atomic_read(&compound_head(page)->_count);
  337. }
  338. static inline void get_huge_page_tail(struct page *page)
  339. {
  340. /*
  341. * __split_huge_page_refcount() cannot run
  342. * from under us.
  343. */
  344. VM_BUG_ON(page_mapcount(page) < 0);
  345. VM_BUG_ON(atomic_read(&page->_count) != 0);
  346. atomic_inc(&page->_mapcount);
  347. }
  348. extern bool __get_page_tail(struct page *page);
  349. static inline void get_page(struct page *page)
  350. {
  351. if (unlikely(PageTail(page)))
  352. if (likely(__get_page_tail(page)))
  353. return;
  354. /*
  355. * Getting a normal page or the head of a compound page
  356. * requires to already have an elevated page->_count.
  357. */
  358. VM_BUG_ON(atomic_read(&page->_count) <= 0);
  359. atomic_inc(&page->_count);
  360. }
  361. static inline struct page *virt_to_head_page(const void *x)
  362. {
  363. struct page *page = virt_to_page(x);
  364. return compound_head(page);
  365. }
  366. /*
  367. * Setup the page count before being freed into the page allocator for
  368. * the first time (boot or memory hotplug)
  369. */
  370. static inline void init_page_count(struct page *page)
  371. {
  372. atomic_set(&page->_count, 1);
  373. }
  374. /*
  375. * PageBuddy() indicate that the page is free and in the buddy system
  376. * (see mm/page_alloc.c).
  377. *
  378. * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
  379. * -2 so that an underflow of the page_mapcount() won't be mistaken
  380. * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
  381. * efficiently by most CPU architectures.
  382. */
  383. #define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
  384. static inline int PageBuddy(struct page *page)
  385. {
  386. return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
  387. }
  388. static inline void __SetPageBuddy(struct page *page)
  389. {
  390. VM_BUG_ON(atomic_read(&page->_mapcount) != -1);
  391. atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
  392. }
  393. static inline void __ClearPageBuddy(struct page *page)
  394. {
  395. VM_BUG_ON(!PageBuddy(page));
  396. atomic_set(&page->_mapcount, -1);
  397. }
  398. void put_page(struct page *page);
  399. void put_pages_list(struct list_head *pages);
  400. void split_page(struct page *page, unsigned int order);
  401. int split_free_page(struct page *page);
  402. /*
  403. * Compound pages have a destructor function. Provide a
  404. * prototype for that function and accessor functions.
  405. * These are _only_ valid on the head of a PG_compound page.
  406. */
  407. typedef void compound_page_dtor(struct page *);
  408. static inline void set_compound_page_dtor(struct page *page,
  409. compound_page_dtor *dtor)
  410. {
  411. page[1].lru.next = (void *)dtor;
  412. }
  413. static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
  414. {
  415. return (compound_page_dtor *)page[1].lru.next;
  416. }
  417. static inline int compound_order(struct page *page)
  418. {
  419. if (!PageHead(page))
  420. return 0;
  421. return (unsigned long)page[1].lru.prev;
  422. }
  423. static inline void set_compound_order(struct page *page, unsigned long order)
  424. {
  425. page[1].lru.prev = (void *)order;
  426. }
  427. #ifdef CONFIG_MMU
  428. /*
  429. * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
  430. * servicing faults for write access. In the normal case, do always want
  431. * pte_mkwrite. But get_user_pages can cause write faults for mappings
  432. * that do not have writing enabled, when used by access_process_vm.
  433. */
  434. static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  435. {
  436. if (likely(vma->vm_flags & VM_WRITE))
  437. pte = pte_mkwrite(pte);
  438. return pte;
  439. }
  440. #endif
  441. /*
  442. * Multiple processes may "see" the same page. E.g. for untouched
  443. * mappings of /dev/null, all processes see the same page full of
  444. * zeroes, and text pages of executables and shared libraries have
  445. * only one copy in memory, at most, normally.
  446. *
  447. * For the non-reserved pages, page_count(page) denotes a reference count.
  448. * page_count() == 0 means the page is free. page->lru is then used for
  449. * freelist management in the buddy allocator.
  450. * page_count() > 0 means the page has been allocated.
  451. *
  452. * Pages are allocated by the slab allocator in order to provide memory
  453. * to kmalloc and kmem_cache_alloc. In this case, the management of the
  454. * page, and the fields in 'struct page' are the responsibility of mm/slab.c
  455. * unless a particular usage is carefully commented. (the responsibility of
  456. * freeing the kmalloc memory is the caller's, of course).
  457. *
  458. * A page may be used by anyone else who does a __get_free_page().
  459. * In this case, page_count still tracks the references, and should only
  460. * be used through the normal accessor functions. The top bits of page->flags
  461. * and page->virtual store page management information, but all other fields
  462. * are unused and could be used privately, carefully. The management of this
  463. * page is the responsibility of the one who allocated it, and those who have
  464. * subsequently been given references to it.
  465. *
  466. * The other pages (we may call them "pagecache pages") are completely
  467. * managed by the Linux memory manager: I/O, buffers, swapping etc.
  468. * The following discussion applies only to them.
  469. *
  470. * A pagecache page contains an opaque `private' member, which belongs to the
  471. * page's address_space. Usually, this is the address of a circular list of
  472. * the page's disk buffers. PG_private must be set to tell the VM to call
  473. * into the filesystem to release these pages.
  474. *
  475. * A page may belong to an inode's memory mapping. In this case, page->mapping
  476. * is the pointer to the inode, and page->index is the file offset of the page,
  477. * in units of PAGE_CACHE_SIZE.
  478. *
  479. * If pagecache pages are not associated with an inode, they are said to be
  480. * anonymous pages. These may become associated with the swapcache, and in that
  481. * case PG_swapcache is set, and page->private is an offset into the swapcache.
  482. *
  483. * In either case (swapcache or inode backed), the pagecache itself holds one
  484. * reference to the page. Setting PG_private should also increment the
  485. * refcount. The each user mapping also has a reference to the page.
  486. *
  487. * The pagecache pages are stored in a per-mapping radix tree, which is
  488. * rooted at mapping->page_tree, and indexed by offset.
  489. * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
  490. * lists, we instead now tag pages as dirty/writeback in the radix tree.
  491. *
  492. * All pagecache pages may be subject to I/O:
  493. * - inode pages may need to be read from disk,
  494. * - inode pages which have been modified and are MAP_SHARED may need
  495. * to be written back to the inode on disk,
  496. * - anonymous pages (including MAP_PRIVATE file mappings) which have been
  497. * modified may need to be swapped out to swap space and (later) to be read
  498. * back into memory.
  499. */
  500. /*
  501. * The zone field is never updated after free_area_init_core()
  502. * sets it, so none of the operations on it need to be atomic.
  503. */
  504. /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NIDPID] | ... | FLAGS | */
  505. #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
  506. #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
  507. #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
  508. #define LAST_NIDPID_PGOFF (ZONES_PGOFF - LAST_NIDPID_WIDTH)
  509. /*
  510. * Define the bit shifts to access each section. For non-existent
  511. * sections we define the shift as 0; that plus a 0 mask ensures
  512. * the compiler will optimise away reference to them.
  513. */
  514. #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
  515. #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
  516. #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
  517. #define LAST_NIDPID_PGSHIFT (LAST_NIDPID_PGOFF * (LAST_NIDPID_WIDTH != 0))
  518. /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
  519. #ifdef NODE_NOT_IN_PAGE_FLAGS
  520. #define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
  521. #define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
  522. SECTIONS_PGOFF : ZONES_PGOFF)
  523. #else
  524. #define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
  525. #define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
  526. NODES_PGOFF : ZONES_PGOFF)
  527. #endif
  528. #define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
  529. #if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  530. #error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  531. #endif
  532. #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
  533. #define NODES_MASK ((1UL << NODES_WIDTH) - 1)
  534. #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
  535. #define LAST_NIDPID_MASK ((1UL << LAST_NIDPID_WIDTH) - 1)
  536. #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
  537. static inline enum zone_type page_zonenum(const struct page *page)
  538. {
  539. return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
  540. }
  541. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  542. #define SECTION_IN_PAGE_FLAGS
  543. #endif
  544. /*
  545. * The identification function is mainly used by the buddy allocator for
  546. * determining if two pages could be buddies. We are not really identifying
  547. * the zone since we could be using the section number id if we do not have
  548. * node id available in page flags.
  549. * We only guarantee that it will return the same value for two combinable
  550. * pages in a zone.
  551. */
  552. static inline int page_zone_id(struct page *page)
  553. {
  554. return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
  555. }
  556. static inline int zone_to_nid(struct zone *zone)
  557. {
  558. #ifdef CONFIG_NUMA
  559. return zone->node;
  560. #else
  561. return 0;
  562. #endif
  563. }
  564. #ifdef NODE_NOT_IN_PAGE_FLAGS
  565. extern int page_to_nid(const struct page *page);
  566. #else
  567. static inline int page_to_nid(const struct page *page)
  568. {
  569. return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
  570. }
  571. #endif
  572. #ifdef CONFIG_NUMA_BALANCING
  573. static inline int nid_pid_to_nidpid(int nid, int pid)
  574. {
  575. return ((nid & LAST__NID_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
  576. }
  577. static inline int nidpid_to_pid(int nidpid)
  578. {
  579. return nidpid & LAST__PID_MASK;
  580. }
  581. static inline int nidpid_to_nid(int nidpid)
  582. {
  583. return (nidpid >> LAST__PID_SHIFT) & LAST__NID_MASK;
  584. }
  585. static inline bool nidpid_pid_unset(int nidpid)
  586. {
  587. return nidpid_to_pid(nidpid) == (-1 & LAST__PID_MASK);
  588. }
  589. static inline bool nidpid_nid_unset(int nidpid)
  590. {
  591. return nidpid_to_nid(nidpid) == (-1 & LAST__NID_MASK);
  592. }
  593. #ifdef LAST_NIDPID_NOT_IN_PAGE_FLAGS
  594. static inline int page_nidpid_xchg_last(struct page *page, int nid)
  595. {
  596. return xchg(&page->_last_nidpid, nid);
  597. }
  598. static inline int page_nidpid_last(struct page *page)
  599. {
  600. return page->_last_nidpid;
  601. }
  602. static inline void page_nidpid_reset_last(struct page *page)
  603. {
  604. page->_last_nidpid = -1;
  605. }
  606. #else
  607. static inline int page_nidpid_last(struct page *page)
  608. {
  609. return (page->flags >> LAST_NIDPID_PGSHIFT) & LAST_NIDPID_MASK;
  610. }
  611. extern int page_nidpid_xchg_last(struct page *page, int nidpid);
  612. static inline void page_nidpid_reset_last(struct page *page)
  613. {
  614. int nidpid = (1 << LAST_NIDPID_SHIFT) - 1;
  615. page->flags &= ~(LAST_NIDPID_MASK << LAST_NIDPID_PGSHIFT);
  616. page->flags |= (nidpid & LAST_NIDPID_MASK) << LAST_NIDPID_PGSHIFT;
  617. }
  618. #endif /* LAST_NIDPID_NOT_IN_PAGE_FLAGS */
  619. #else
  620. static inline int page_nidpid_xchg_last(struct page *page, int nidpid)
  621. {
  622. return page_to_nid(page);
  623. }
  624. static inline int page_nidpid_last(struct page *page)
  625. {
  626. return page_to_nid(page);
  627. }
  628. static inline int nidpid_to_nid(int nidpid)
  629. {
  630. return -1;
  631. }
  632. static inline int nidpid_to_pid(int nidpid)
  633. {
  634. return -1;
  635. }
  636. static inline int nid_pid_to_nidpid(int nid, int pid)
  637. {
  638. return -1;
  639. }
  640. static inline bool nidpid_pid_unset(int nidpid)
  641. {
  642. return 1;
  643. }
  644. static inline void page_nidpid_reset_last(struct page *page)
  645. {
  646. }
  647. #endif
  648. static inline struct zone *page_zone(const struct page *page)
  649. {
  650. return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
  651. }
  652. #ifdef SECTION_IN_PAGE_FLAGS
  653. static inline void set_page_section(struct page *page, unsigned long section)
  654. {
  655. page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
  656. page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
  657. }
  658. static inline unsigned long page_to_section(const struct page *page)
  659. {
  660. return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
  661. }
  662. #endif
  663. static inline void set_page_zone(struct page *page, enum zone_type zone)
  664. {
  665. page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
  666. page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
  667. }
  668. static inline void set_page_node(struct page *page, unsigned long node)
  669. {
  670. page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
  671. page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
  672. }
  673. static inline void set_page_links(struct page *page, enum zone_type zone,
  674. unsigned long node, unsigned long pfn)
  675. {
  676. set_page_zone(page, zone);
  677. set_page_node(page, node);
  678. #ifdef SECTION_IN_PAGE_FLAGS
  679. set_page_section(page, pfn_to_section_nr(pfn));
  680. #endif
  681. }
  682. /*
  683. * Some inline functions in vmstat.h depend on page_zone()
  684. */
  685. #include <linux/vmstat.h>
  686. static __always_inline void *lowmem_page_address(const struct page *page)
  687. {
  688. return __va(PFN_PHYS(page_to_pfn(page)));
  689. }
  690. #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
  691. #define HASHED_PAGE_VIRTUAL
  692. #endif
  693. #if defined(WANT_PAGE_VIRTUAL)
  694. #define page_address(page) ((page)->virtual)
  695. #define set_page_address(page, address) \
  696. do { \
  697. (page)->virtual = (address); \
  698. } while(0)
  699. #define page_address_init() do { } while(0)
  700. #endif
  701. #if defined(HASHED_PAGE_VIRTUAL)
  702. void *page_address(const struct page *page);
  703. void set_page_address(struct page *page, void *virtual);
  704. void page_address_init(void);
  705. #endif
  706. #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
  707. #define page_address(page) lowmem_page_address(page)
  708. #define set_page_address(page, address) do { } while(0)
  709. #define page_address_init() do { } while(0)
  710. #endif
  711. /*
  712. * On an anonymous page mapped into a user virtual memory area,
  713. * page->mapping points to its anon_vma, not to a struct address_space;
  714. * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
  715. *
  716. * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
  717. * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
  718. * and then page->mapping points, not to an anon_vma, but to a private
  719. * structure which KSM associates with that merged page. See ksm.h.
  720. *
  721. * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
  722. *
  723. * Please note that, confusingly, "page_mapping" refers to the inode
  724. * address_space which maps the page from disk; whereas "page_mapped"
  725. * refers to user virtual address space into which the page is mapped.
  726. */
  727. #define PAGE_MAPPING_ANON 1
  728. #define PAGE_MAPPING_KSM 2
  729. #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
  730. extern struct address_space *page_mapping(struct page *page);
  731. /* Neutral page->mapping pointer to address_space or anon_vma or other */
  732. static inline void *page_rmapping(struct page *page)
  733. {
  734. return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);
  735. }
  736. extern struct address_space *__page_file_mapping(struct page *);
  737. static inline
  738. struct address_space *page_file_mapping(struct page *page)
  739. {
  740. if (unlikely(PageSwapCache(page)))
  741. return __page_file_mapping(page);
  742. return page->mapping;
  743. }
  744. static inline int PageAnon(struct page *page)
  745. {
  746. return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
  747. }
  748. /*
  749. * Return the pagecache index of the passed page. Regular pagecache pages
  750. * use ->index whereas swapcache pages use ->private
  751. */
  752. static inline pgoff_t page_index(struct page *page)
  753. {
  754. if (unlikely(PageSwapCache(page)))
  755. return page_private(page);
  756. return page->index;
  757. }
  758. extern pgoff_t __page_file_index(struct page *page);
  759. /*
  760. * Return the file index of the page. Regular pagecache pages use ->index
  761. * whereas swapcache pages use swp_offset(->private)
  762. */
  763. static inline pgoff_t page_file_index(struct page *page)
  764. {
  765. if (unlikely(PageSwapCache(page)))
  766. return __page_file_index(page);
  767. return page->index;
  768. }
  769. /*
  770. * Return true if this page is mapped into pagetables.
  771. */
  772. static inline int page_mapped(struct page *page)
  773. {
  774. return atomic_read(&(page)->_mapcount) >= 0;
  775. }
  776. /*
  777. * Different kinds of faults, as returned by handle_mm_fault().
  778. * Used to decide whether a process gets delivered SIGBUS or
  779. * just gets major/minor fault counters bumped up.
  780. */
  781. #define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
  782. #define VM_FAULT_OOM 0x0001
  783. #define VM_FAULT_SIGBUS 0x0002
  784. #define VM_FAULT_MAJOR 0x0004
  785. #define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
  786. #define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
  787. #define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
  788. #define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
  789. #define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
  790. #define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
  791. #define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
  792. #define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
  793. #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \
  794. VM_FAULT_FALLBACK | VM_FAULT_HWPOISON_LARGE)
  795. /* Encode hstate index for a hwpoisoned large page */
  796. #define VM_FAULT_SET_HINDEX(x) ((x) << 12)
  797. #define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
  798. /*
  799. * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
  800. */
  801. extern void pagefault_out_of_memory(void);
  802. #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
  803. /*
  804. * Flags passed to show_mem() and show_free_areas() to suppress output in
  805. * various contexts.
  806. */
  807. #define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
  808. #define SHOW_MEM_FILTER_PAGE_COUNT (0x0002u) /* page type count */
  809. extern void show_free_areas(unsigned int flags);
  810. extern bool skip_free_areas_node(unsigned int flags, int nid);
  811. int shmem_zero_setup(struct vm_area_struct *);
  812. extern int can_do_mlock(void);
  813. extern int user_shm_lock(size_t, struct user_struct *);
  814. extern void user_shm_unlock(size_t, struct user_struct *);
  815. /*
  816. * Parameter block passed down to zap_pte_range in exceptional cases.
  817. */
  818. struct zap_details {
  819. struct vm_area_struct *nonlinear_vma; /* Check page->index if set */
  820. struct address_space *check_mapping; /* Check page->mapping if set */
  821. pgoff_t first_index; /* Lowest page->index to unmap */
  822. pgoff_t last_index; /* Highest page->index to unmap */
  823. };
  824. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  825. pte_t pte);
  826. int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  827. unsigned long size);
  828. void zap_page_range(struct vm_area_struct *vma, unsigned long address,
  829. unsigned long size, struct zap_details *);
  830. void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
  831. unsigned long start, unsigned long end);
  832. /**
  833. * mm_walk - callbacks for walk_page_range
  834. * @pgd_entry: if set, called for each non-empty PGD (top-level) entry
  835. * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry
  836. * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
  837. * this handler is required to be able to handle
  838. * pmd_trans_huge() pmds. They may simply choose to
  839. * split_huge_page() instead of handling it explicitly.
  840. * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
  841. * @pte_hole: if set, called for each hole at all levels
  842. * @hugetlb_entry: if set, called for each hugetlb entry
  843. * *Caution*: The caller must hold mmap_sem() if @hugetlb_entry
  844. * is used.
  845. *
  846. * (see walk_page_range for more details)
  847. */
  848. struct mm_walk {
  849. int (*pgd_entry)(pgd_t *pgd, unsigned long addr,
  850. unsigned long next, struct mm_walk *walk);
  851. int (*pud_entry)(pud_t *pud, unsigned long addr,
  852. unsigned long next, struct mm_walk *walk);
  853. int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
  854. unsigned long next, struct mm_walk *walk);
  855. int (*pte_entry)(pte_t *pte, unsigned long addr,
  856. unsigned long next, struct mm_walk *walk);
  857. int (*pte_hole)(unsigned long addr, unsigned long next,
  858. struct mm_walk *walk);
  859. int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
  860. unsigned long addr, unsigned long next,
  861. struct mm_walk *walk);
  862. struct mm_struct *mm;
  863. void *private;
  864. };
  865. int walk_page_range(unsigned long addr, unsigned long end,
  866. struct mm_walk *walk);
  867. void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
  868. unsigned long end, unsigned long floor, unsigned long ceiling);
  869. int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
  870. struct vm_area_struct *vma);
  871. void unmap_mapping_range(struct address_space *mapping,
  872. loff_t const holebegin, loff_t const holelen, int even_cows);
  873. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  874. unsigned long *pfn);
  875. int follow_phys(struct vm_area_struct *vma, unsigned long address,
  876. unsigned int flags, unsigned long *prot, resource_size_t *phys);
  877. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  878. void *buf, int len, int write);
  879. static inline void unmap_shared_mapping_range(struct address_space *mapping,
  880. loff_t const holebegin, loff_t const holelen)
  881. {
  882. unmap_mapping_range(mapping, holebegin, holelen, 0);
  883. }
  884. extern void truncate_pagecache(struct inode *inode, loff_t new);
  885. extern void truncate_setsize(struct inode *inode, loff_t newsize);
  886. void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
  887. int truncate_inode_page(struct address_space *mapping, struct page *page);
  888. int generic_error_remove_page(struct address_space *mapping, struct page *page);
  889. int invalidate_inode_page(struct page *page);
  890. #ifdef CONFIG_MMU
  891. extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  892. unsigned long address, unsigned int flags);
  893. extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
  894. unsigned long address, unsigned int fault_flags);
  895. #else
  896. static inline int handle_mm_fault(struct mm_struct *mm,
  897. struct vm_area_struct *vma, unsigned long address,
  898. unsigned int flags)
  899. {
  900. /* should never happen if there's no MMU */
  901. BUG();
  902. return VM_FAULT_SIGBUS;
  903. }
  904. static inline int fixup_user_fault(struct task_struct *tsk,
  905. struct mm_struct *mm, unsigned long address,
  906. unsigned int fault_flags)
  907. {
  908. /* should never happen if there's no MMU */
  909. BUG();
  910. return -EFAULT;
  911. }
  912. #endif
  913. extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
  914. extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  915. void *buf, int len, int write);
  916. long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  917. unsigned long start, unsigned long nr_pages,
  918. unsigned int foll_flags, struct page **pages,
  919. struct vm_area_struct **vmas, int *nonblocking);
  920. long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  921. unsigned long start, unsigned long nr_pages,
  922. int write, int force, struct page **pages,
  923. struct vm_area_struct **vmas);
  924. int get_user_pages_fast(unsigned long start, int nr_pages, int write,
  925. struct page **pages);
  926. struct kvec;
  927. int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
  928. struct page **pages);
  929. int get_kernel_page(unsigned long start, int write, struct page **pages);
  930. struct page *get_dump_page(unsigned long addr);
  931. extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
  932. extern void do_invalidatepage(struct page *page, unsigned int offset,
  933. unsigned int length);
  934. int __set_page_dirty_nobuffers(struct page *page);
  935. int __set_page_dirty_no_writeback(struct page *page);
  936. int redirty_page_for_writepage(struct writeback_control *wbc,
  937. struct page *page);
  938. void account_page_dirtied(struct page *page, struct address_space *mapping);
  939. void account_page_writeback(struct page *page);
  940. int set_page_dirty(struct page *page);
  941. int set_page_dirty_lock(struct page *page);
  942. int clear_page_dirty_for_io(struct page *page);
  943. /* Is the vma a continuation of the stack vma above it? */
  944. static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
  945. {
  946. return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
  947. }
  948. static inline int stack_guard_page_start(struct vm_area_struct *vma,
  949. unsigned long addr)
  950. {
  951. return (vma->vm_flags & VM_GROWSDOWN) &&
  952. (vma->vm_start == addr) &&
  953. !vma_growsdown(vma->vm_prev, addr);
  954. }
  955. /* Is the vma a continuation of the stack vma below it? */
  956. static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
  957. {
  958. return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
  959. }
  960. static inline int stack_guard_page_end(struct vm_area_struct *vma,
  961. unsigned long addr)
  962. {
  963. return (vma->vm_flags & VM_GROWSUP) &&
  964. (vma->vm_end == addr) &&
  965. !vma_growsup(vma->vm_next, addr);
  966. }
  967. extern pid_t
  968. vm_is_stack(struct task_struct *task, struct vm_area_struct *vma, int in_group);
  969. extern unsigned long move_page_tables(struct vm_area_struct *vma,
  970. unsigned long old_addr, struct vm_area_struct *new_vma,
  971. unsigned long new_addr, unsigned long len,
  972. bool need_rmap_locks);
  973. extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
  974. unsigned long end, pgprot_t newprot,
  975. int dirty_accountable, int prot_numa);
  976. extern int mprotect_fixup(struct vm_area_struct *vma,
  977. struct vm_area_struct **pprev, unsigned long start,
  978. unsigned long end, unsigned long newflags);
  979. /*
  980. * doesn't attempt to fault and will return short.
  981. */
  982. int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
  983. struct page **pages);
  984. /*
  985. * per-process(per-mm_struct) statistics.
  986. */
  987. static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
  988. {
  989. long val = atomic_long_read(&mm->rss_stat.count[member]);
  990. #ifdef SPLIT_RSS_COUNTING
  991. /*
  992. * counter is updated in asynchronous manner and may go to minus.
  993. * But it's never be expected number for users.
  994. */
  995. if (val < 0)
  996. val = 0;
  997. #endif
  998. return (unsigned long)val;
  999. }
  1000. static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
  1001. {
  1002. atomic_long_add(value, &mm->rss_stat.count[member]);
  1003. }
  1004. static inline void inc_mm_counter(struct mm_struct *mm, int member)
  1005. {
  1006. atomic_long_inc(&mm->rss_stat.count[member]);
  1007. }
  1008. static inline void dec_mm_counter(struct mm_struct *mm, int member)
  1009. {
  1010. atomic_long_dec(&mm->rss_stat.count[member]);
  1011. }
  1012. static inline unsigned long get_mm_rss(struct mm_struct *mm)
  1013. {
  1014. return get_mm_counter(mm, MM_FILEPAGES) +
  1015. get_mm_counter(mm, MM_ANONPAGES);
  1016. }
  1017. static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
  1018. {
  1019. return max(mm->hiwater_rss, get_mm_rss(mm));
  1020. }
  1021. static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
  1022. {
  1023. return max(mm->hiwater_vm, mm->total_vm);
  1024. }
  1025. static inline void update_hiwater_rss(struct mm_struct *mm)
  1026. {
  1027. unsigned long _rss = get_mm_rss(mm);
  1028. if ((mm)->hiwater_rss < _rss)
  1029. (mm)->hiwater_rss = _rss;
  1030. }
  1031. static inline void update_hiwater_vm(struct mm_struct *mm)
  1032. {
  1033. if (mm->hiwater_vm < mm->total_vm)
  1034. mm->hiwater_vm = mm->total_vm;
  1035. }
  1036. static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
  1037. struct mm_struct *mm)
  1038. {
  1039. unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
  1040. if (*maxrss < hiwater_rss)
  1041. *maxrss = hiwater_rss;
  1042. }
  1043. #if defined(SPLIT_RSS_COUNTING)
  1044. void sync_mm_rss(struct mm_struct *mm);
  1045. #else
  1046. static inline void sync_mm_rss(struct mm_struct *mm)
  1047. {
  1048. }
  1049. #endif
  1050. int vma_wants_writenotify(struct vm_area_struct *vma);
  1051. extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1052. spinlock_t **ptl);
  1053. static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1054. spinlock_t **ptl)
  1055. {
  1056. pte_t *ptep;
  1057. __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
  1058. return ptep;
  1059. }
  1060. #ifdef __PAGETABLE_PUD_FOLDED
  1061. static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
  1062. unsigned long address)
  1063. {
  1064. return 0;
  1065. }
  1066. #else
  1067. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
  1068. #endif
  1069. #ifdef __PAGETABLE_PMD_FOLDED
  1070. static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
  1071. unsigned long address)
  1072. {
  1073. return 0;
  1074. }
  1075. #else
  1076. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
  1077. #endif
  1078. int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
  1079. pmd_t *pmd, unsigned long address);
  1080. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
  1081. /*
  1082. * The following ifdef needed to get the 4level-fixup.h header to work.
  1083. * Remove it when 4level-fixup.h has been removed.
  1084. */
  1085. #if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
  1086. static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  1087. {
  1088. return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
  1089. NULL: pud_offset(pgd, address);
  1090. }
  1091. static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  1092. {
  1093. return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
  1094. NULL: pmd_offset(pud, address);
  1095. }
  1096. #endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
  1097. #if USE_SPLIT_PTLOCKS
  1098. /*
  1099. * We tuck a spinlock to guard each pagetable page into its struct page,
  1100. * at page->private, with BUILD_BUG_ON to make sure that this will not
  1101. * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
  1102. * When freeing, reset page->mapping so free_pages_check won't complain.
  1103. */
  1104. #define __pte_lockptr(page) &((page)->ptl)
  1105. #define pte_lock_init(_page) do { \
  1106. spin_lock_init(__pte_lockptr(_page)); \
  1107. } while (0)
  1108. #define pte_lock_deinit(page) ((page)->mapping = NULL)
  1109. #define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
  1110. #else /* !USE_SPLIT_PTLOCKS */
  1111. /*
  1112. * We use mm->page_table_lock to guard all pagetable pages of the mm.
  1113. */
  1114. #define pte_lock_init(page) do {} while (0)
  1115. #define pte_lock_deinit(page) do {} while (0)
  1116. #define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
  1117. #endif /* USE_SPLIT_PTLOCKS */
  1118. static inline void pgtable_page_ctor(struct page *page)
  1119. {
  1120. pte_lock_init(page);
  1121. inc_zone_page_state(page, NR_PAGETABLE);
  1122. }
  1123. static inline void pgtable_page_dtor(struct page *page)
  1124. {
  1125. pte_lock_deinit(page);
  1126. dec_zone_page_state(page, NR_PAGETABLE);
  1127. }
  1128. #define pte_offset_map_lock(mm, pmd, address, ptlp) \
  1129. ({ \
  1130. spinlock_t *__ptl = pte_lockptr(mm, pmd); \
  1131. pte_t *__pte = pte_offset_map(pmd, address); \
  1132. *(ptlp) = __ptl; \
  1133. spin_lock(__ptl); \
  1134. __pte; \
  1135. })
  1136. #define pte_unmap_unlock(pte, ptl) do { \
  1137. spin_unlock(ptl); \
  1138. pte_unmap(pte); \
  1139. } while (0)
  1140. #define pte_alloc_map(mm, vma, pmd, address) \
  1141. ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
  1142. pmd, address))? \
  1143. NULL: pte_offset_map(pmd, address))
  1144. #define pte_alloc_map_lock(mm, pmd, address, ptlp) \
  1145. ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
  1146. pmd, address))? \
  1147. NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
  1148. #define pte_alloc_kernel(pmd, address) \
  1149. ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
  1150. NULL: pte_offset_kernel(pmd, address))
  1151. extern void free_area_init(unsigned long * zones_size);
  1152. extern void free_area_init_node(int nid, unsigned long * zones_size,
  1153. unsigned long zone_start_pfn, unsigned long *zholes_size);
  1154. extern void free_initmem(void);
  1155. /*
  1156. * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
  1157. * into the buddy system. The freed pages will be poisoned with pattern
  1158. * "poison" if it's within range [0, UCHAR_MAX].
  1159. * Return pages freed into the buddy system.
  1160. */
  1161. extern unsigned long free_reserved_area(void *start, void *end,
  1162. int poison, char *s);
  1163. #ifdef CONFIG_HIGHMEM
  1164. /*
  1165. * Free a highmem page into the buddy system, adjusting totalhigh_pages
  1166. * and totalram_pages.
  1167. */
  1168. extern void free_highmem_page(struct page *page);
  1169. #endif
  1170. extern void adjust_managed_page_count(struct page *page, long count);
  1171. extern void mem_init_print_info(const char *str);
  1172. /* Free the reserved page into the buddy system, so it gets managed. */
  1173. static inline void __free_reserved_page(struct page *page)
  1174. {
  1175. ClearPageReserved(page);
  1176. init_page_count(page);
  1177. __free_page(page);
  1178. }
  1179. static inline void free_reserved_page(struct page *page)
  1180. {
  1181. __free_reserved_page(page);
  1182. adjust_managed_page_count(page, 1);
  1183. }
  1184. static inline void mark_page_reserved(struct page *page)
  1185. {
  1186. SetPageReserved(page);
  1187. adjust_managed_page_count(page, -1);
  1188. }
  1189. /*
  1190. * Default method to free all the __init memory into the buddy system.
  1191. * The freed pages will be poisoned with pattern "poison" if it's within
  1192. * range [0, UCHAR_MAX].
  1193. * Return pages freed into the buddy system.
  1194. */
  1195. static inline unsigned long free_initmem_default(int poison)
  1196. {
  1197. extern char __init_begin[], __init_end[];
  1198. return free_reserved_area(&__init_begin, &__init_end,
  1199. poison, "unused kernel");
  1200. }
  1201. static inline unsigned long get_num_physpages(void)
  1202. {
  1203. int nid;
  1204. unsigned long phys_pages = 0;
  1205. for_each_online_node(nid)
  1206. phys_pages += node_present_pages(nid);
  1207. return phys_pages;
  1208. }
  1209. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  1210. /*
  1211. * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
  1212. * zones, allocate the backing mem_map and account for memory holes in a more
  1213. * architecture independent manner. This is a substitute for creating the
  1214. * zone_sizes[] and zholes_size[] arrays and passing them to
  1215. * free_area_init_node()
  1216. *
  1217. * An architecture is expected to register range of page frames backed by
  1218. * physical memory with memblock_add[_node]() before calling
  1219. * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
  1220. * usage, an architecture is expected to do something like
  1221. *
  1222. * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
  1223. * max_highmem_pfn};
  1224. * for_each_valid_physical_page_range()
  1225. * memblock_add_node(base, size, nid)
  1226. * free_area_init_nodes(max_zone_pfns);
  1227. *
  1228. * free_bootmem_with_active_regions() calls free_bootmem_node() for each
  1229. * registered physical page range. Similarly
  1230. * sparse_memory_present_with_active_regions() calls memory_present() for
  1231. * each range when SPARSEMEM is enabled.
  1232. *
  1233. * See mm/page_alloc.c for more information on each function exposed by
  1234. * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
  1235. */
  1236. extern void free_area_init_nodes(unsigned long *max_zone_pfn);
  1237. unsigned long node_map_pfn_alignment(void);
  1238. unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
  1239. unsigned long end_pfn);
  1240. extern unsigned long absent_pages_in_range(unsigned long start_pfn,
  1241. unsigned long end_pfn);
  1242. extern void get_pfn_range_for_nid(unsigned int nid,
  1243. unsigned long *start_pfn, unsigned long *end_pfn);
  1244. extern unsigned long find_min_pfn_with_active_regions(void);
  1245. extern void free_bootmem_with_active_regions(int nid,
  1246. unsigned long max_low_pfn);
  1247. extern void sparse_memory_present_with_active_regions(int nid);
  1248. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  1249. #if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
  1250. !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
  1251. static inline int __early_pfn_to_nid(unsigned long pfn)
  1252. {
  1253. return 0;
  1254. }
  1255. #else
  1256. /* please see mm/page_alloc.c */
  1257. extern int __meminit early_pfn_to_nid(unsigned long pfn);
  1258. #ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  1259. /* there is a per-arch backend function. */
  1260. extern int __meminit __early_pfn_to_nid(unsigned long pfn);
  1261. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  1262. #endif
  1263. extern void set_dma_reserve(unsigned long new_dma_reserve);
  1264. extern void memmap_init_zone(unsigned long, int, unsigned long,
  1265. unsigned long, enum memmap_context);
  1266. extern void setup_per_zone_wmarks(void);
  1267. extern int __meminit init_per_zone_wmark_min(void);
  1268. extern void mem_init(void);
  1269. extern void __init mmap_init(void);
  1270. extern void show_mem(unsigned int flags);
  1271. extern void si_meminfo(struct sysinfo * val);
  1272. extern void si_meminfo_node(struct sysinfo *val, int nid);
  1273. extern __printf(3, 4)
  1274. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
  1275. extern void setup_per_cpu_pageset(void);
  1276. extern void zone_pcp_update(struct zone *zone);
  1277. extern void zone_pcp_reset(struct zone *zone);
  1278. /* page_alloc.c */
  1279. extern int min_free_kbytes;
  1280. /* nommu.c */
  1281. extern atomic_long_t mmap_pages_allocated;
  1282. extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
  1283. /* interval_tree.c */
  1284. void vma_interval_tree_insert(struct vm_area_struct *node,
  1285. struct rb_root *root);
  1286. void vma_interval_tree_insert_after(struct vm_area_struct *node,
  1287. struct vm_area_struct *prev,
  1288. struct rb_root *root);
  1289. void vma_interval_tree_remove(struct vm_area_struct *node,
  1290. struct rb_root *root);
  1291. struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
  1292. unsigned long start, unsigned long last);
  1293. struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
  1294. unsigned long start, unsigned long last);
  1295. #define vma_interval_tree_foreach(vma, root, start, last) \
  1296. for (vma = vma_interval_tree_iter_first(root, start, last); \
  1297. vma; vma = vma_interval_tree_iter_next(vma, start, last))
  1298. static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
  1299. struct list_head *list)
  1300. {
  1301. list_add_tail(&vma->shared.nonlinear, list);
  1302. }
  1303. void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
  1304. struct rb_root *root);
  1305. void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
  1306. struct rb_root *root);
  1307. struct anon_vma_chain *anon_vma_interval_tree_iter_first(
  1308. struct rb_root *root, unsigned long start, unsigned long last);
  1309. struct anon_vma_chain *anon_vma_interval_tree_iter_next(
  1310. struct anon_vma_chain *node, unsigned long start, unsigned long last);
  1311. #ifdef CONFIG_DEBUG_VM_RB
  1312. void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
  1313. #endif
  1314. #define anon_vma_interval_tree_foreach(avc, root, start, last) \
  1315. for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
  1316. avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
  1317. /* mmap.c */
  1318. extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
  1319. extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
  1320. unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
  1321. extern struct vm_area_struct *vma_merge(struct mm_struct *,
  1322. struct vm_area_struct *prev, unsigned long addr, unsigned long end,
  1323. unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
  1324. struct mempolicy *);
  1325. extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
  1326. extern int split_vma(struct mm_struct *,
  1327. struct vm_area_struct *, unsigned long addr, int new_below);
  1328. extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
  1329. extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
  1330. struct rb_node **, struct rb_node *);
  1331. extern void unlink_file_vma(struct vm_area_struct *);
  1332. extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
  1333. unsigned long addr, unsigned long len, pgoff_t pgoff,
  1334. bool *need_rmap_locks);
  1335. extern void exit_mmap(struct mm_struct *);
  1336. extern int mm_take_all_locks(struct mm_struct *mm);
  1337. extern void mm_drop_all_locks(struct mm_struct *mm);
  1338. extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
  1339. extern struct file *get_mm_exe_file(struct mm_struct *mm);
  1340. extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
  1341. extern int install_special_mapping(struct mm_struct *mm,
  1342. unsigned long addr, unsigned long len,
  1343. unsigned long flags, struct page **pages);
  1344. extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
  1345. extern unsigned long mmap_region(struct file *file, unsigned long addr,
  1346. unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
  1347. extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
  1348. unsigned long len, unsigned long prot, unsigned long flags,
  1349. unsigned long pgoff, unsigned long *populate);
  1350. extern int do_munmap(struct mm_struct *, unsigned long, size_t);
  1351. #ifdef CONFIG_MMU
  1352. extern int __mm_populate(unsigned long addr, unsigned long len,
  1353. int ignore_errors);
  1354. static inline void mm_populate(unsigned long addr, unsigned long len)
  1355. {
  1356. /* Ignore errors */
  1357. (void) __mm_populate(addr, len, 1);
  1358. }
  1359. #else
  1360. static inline void mm_populate(unsigned long addr, unsigned long len) {}
  1361. #endif
  1362. /* These take the mm semaphore themselves */
  1363. extern unsigned long vm_brk(unsigned long, unsigned long);
  1364. extern int vm_munmap(unsigned long, size_t);
  1365. extern unsigned long vm_mmap(struct file *, unsigned long,
  1366. unsigned long, unsigned long,
  1367. unsigned long, unsigned long);
  1368. struct vm_unmapped_area_info {
  1369. #define VM_UNMAPPED_AREA_TOPDOWN 1
  1370. unsigned long flags;
  1371. unsigned long length;
  1372. unsigned long low_limit;
  1373. unsigned long high_limit;
  1374. unsigned long align_mask;
  1375. unsigned long align_offset;
  1376. };
  1377. extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
  1378. extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
  1379. /*
  1380. * Search for an unmapped address range.
  1381. *
  1382. * We are looking for a range that:
  1383. * - does not intersect with any VMA;
  1384. * - is contained within the [low_limit, high_limit) interval;
  1385. * - is at least the desired size.
  1386. * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
  1387. */
  1388. static inline unsigned long
  1389. vm_unmapped_area(struct vm_unmapped_area_info *info)
  1390. {
  1391. if (!(info->flags & VM_UNMAPPED_AREA_TOPDOWN))
  1392. return unmapped_area(info);
  1393. else
  1394. return unmapped_area_topdown(info);
  1395. }
  1396. /* truncate.c */
  1397. extern void truncate_inode_pages(struct address_space *, loff_t);
  1398. extern void truncate_inode_pages_range(struct address_space *,
  1399. loff_t lstart, loff_t lend);
  1400. /* generic vm_area_ops exported for stackable file systems */
  1401. extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
  1402. extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
  1403. /* mm/page-writeback.c */
  1404. int write_one_page(struct page *page, int wait);
  1405. void task_dirty_inc(struct task_struct *tsk);
  1406. /* readahead.c */
  1407. #define VM_MAX_READAHEAD 128 /* kbytes */
  1408. #define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
  1409. int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
  1410. pgoff_t offset, unsigned long nr_to_read);
  1411. void page_cache_sync_readahead(struct address_space *mapping,
  1412. struct file_ra_state *ra,
  1413. struct file *filp,
  1414. pgoff_t offset,
  1415. unsigned long size);
  1416. void page_cache_async_readahead(struct address_space *mapping,
  1417. struct file_ra_state *ra,
  1418. struct file *filp,
  1419. struct page *pg,
  1420. pgoff_t offset,
  1421. unsigned long size);
  1422. unsigned long max_sane_readahead(unsigned long nr);
  1423. unsigned long ra_submit(struct file_ra_state *ra,
  1424. struct address_space *mapping,
  1425. struct file *filp);
  1426. /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
  1427. extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
  1428. /* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
  1429. extern int expand_downwards(struct vm_area_struct *vma,
  1430. unsigned long address);
  1431. #if VM_GROWSUP
  1432. extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
  1433. #else
  1434. #define expand_upwards(vma, address) do { } while (0)
  1435. #endif
  1436. /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
  1437. extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
  1438. extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
  1439. struct vm_area_struct **pprev);
  1440. /* Look up the first VMA which intersects the interval start_addr..end_addr-1,
  1441. NULL if none. Assume start_addr < end_addr. */
  1442. static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
  1443. {
  1444. struct vm_area_struct * vma = find_vma(mm,start_addr);
  1445. if (vma && end_addr <= vma->vm_start)
  1446. vma = NULL;
  1447. return vma;
  1448. }
  1449. static inline unsigned long vma_pages(struct vm_area_struct *vma)
  1450. {
  1451. return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  1452. }
  1453. /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
  1454. static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
  1455. unsigned long vm_start, unsigned long vm_end)
  1456. {
  1457. struct vm_area_struct *vma = find_vma(mm, vm_start);
  1458. if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
  1459. vma = NULL;
  1460. return vma;
  1461. }
  1462. #ifdef CONFIG_MMU
  1463. pgprot_t vm_get_page_prot(unsigned long vm_flags);
  1464. #else
  1465. static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
  1466. {
  1467. return __pgprot(0);
  1468. }
  1469. #endif
  1470. #ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
  1471. unsigned long change_prot_numa(struct vm_area_struct *vma,
  1472. unsigned long start, unsigned long end);
  1473. #endif
  1474. struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
  1475. int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
  1476. unsigned long pfn, unsigned long size, pgprot_t);
  1477. int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
  1478. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1479. unsigned long pfn);
  1480. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1481. unsigned long pfn);
  1482. int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
  1483. struct page *follow_page_mask(struct vm_area_struct *vma,
  1484. unsigned long address, unsigned int foll_flags,
  1485. unsigned int *page_mask);
  1486. static inline struct page *follow_page(struct vm_area_struct *vma,
  1487. unsigned long address, unsigned int foll_flags)
  1488. {
  1489. unsigned int unused_page_mask;
  1490. return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
  1491. }
  1492. #define FOLL_WRITE 0x01 /* check pte is writable */
  1493. #define FOLL_TOUCH 0x02 /* mark page accessed */
  1494. #define FOLL_GET 0x04 /* do get_page on page */
  1495. #define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
  1496. #define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
  1497. #define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
  1498. * and return without waiting upon it */
  1499. #define FOLL_MLOCK 0x40 /* mark page as mlocked */
  1500. #define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
  1501. #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
  1502. #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
  1503. #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
  1504. typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
  1505. void *data);
  1506. extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
  1507. unsigned long size, pte_fn_t fn, void *data);
  1508. #ifdef CONFIG_PROC_FS
  1509. void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
  1510. #else
  1511. static inline void vm_stat_account(struct mm_struct *mm,
  1512. unsigned long flags, struct file *file, long pages)
  1513. {
  1514. mm->total_vm += pages;
  1515. }
  1516. #endif /* CONFIG_PROC_FS */
  1517. #ifdef CONFIG_DEBUG_PAGEALLOC
  1518. extern void kernel_map_pages(struct page *page, int numpages, int enable);
  1519. #ifdef CONFIG_HIBERNATION
  1520. extern bool kernel_page_present(struct page *page);
  1521. #endif /* CONFIG_HIBERNATION */
  1522. #else
  1523. static inline void
  1524. kernel_map_pages(struct page *page, int numpages, int enable) {}
  1525. #ifdef CONFIG_HIBERNATION
  1526. static inline bool kernel_page_present(struct page *page) { return true; }
  1527. #endif /* CONFIG_HIBERNATION */
  1528. #endif
  1529. extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
  1530. #ifdef __HAVE_ARCH_GATE_AREA
  1531. int in_gate_area_no_mm(unsigned long addr);
  1532. int in_gate_area(struct mm_struct *mm, unsigned long addr);
  1533. #else
  1534. int in_gate_area_no_mm(unsigned long addr);
  1535. #define in_gate_area(mm, addr) ({(void)mm; in_gate_area_no_mm(addr);})
  1536. #endif /* __HAVE_ARCH_GATE_AREA */
  1537. #ifdef CONFIG_SYSCTL
  1538. extern int sysctl_drop_caches;
  1539. int drop_caches_sysctl_handler(struct ctl_table *, int,
  1540. void __user *, size_t *, loff_t *);
  1541. #endif
  1542. unsigned long shrink_slab(struct shrink_control *shrink,
  1543. unsigned long nr_pages_scanned,
  1544. unsigned long lru_pages);
  1545. #ifndef CONFIG_MMU
  1546. #define randomize_va_space 0
  1547. #else
  1548. extern int randomize_va_space;
  1549. #endif
  1550. const char * arch_vma_name(struct vm_area_struct *vma);
  1551. void print_vma_addr(char *prefix, unsigned long rip);
  1552. void sparse_mem_maps_populate_node(struct page **map_map,
  1553. unsigned long pnum_begin,
  1554. unsigned long pnum_end,
  1555. unsigned long map_count,
  1556. int nodeid);
  1557. struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
  1558. pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
  1559. pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
  1560. pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
  1561. pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
  1562. void *vmemmap_alloc_block(unsigned long size, int node);
  1563. void *vmemmap_alloc_block_buf(unsigned long size, int node);
  1564. void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
  1565. int vmemmap_populate_basepages(unsigned long start, unsigned long end,
  1566. int node);
  1567. int vmemmap_populate(unsigned long start, unsigned long end, int node);
  1568. void vmemmap_populate_print_last(void);
  1569. #ifdef CONFIG_MEMORY_HOTPLUG
  1570. void vmemmap_free(unsigned long start, unsigned long end);
  1571. #endif
  1572. void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
  1573. unsigned long size);
  1574. enum mf_flags {
  1575. MF_COUNT_INCREASED = 1 << 0,
  1576. MF_ACTION_REQUIRED = 1 << 1,
  1577. MF_MUST_KILL = 1 << 2,
  1578. MF_SOFT_OFFLINE = 1 << 3,
  1579. };
  1580. extern int memory_failure(unsigned long pfn, int trapno, int flags);
  1581. extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
  1582. extern int unpoison_memory(unsigned long pfn);
  1583. extern int sysctl_memory_failure_early_kill;
  1584. extern int sysctl_memory_failure_recovery;
  1585. extern void shake_page(struct page *p, int access);
  1586. extern atomic_long_t num_poisoned_pages;
  1587. extern int soft_offline_page(struct page *page, int flags);
  1588. extern void dump_page(struct page *page);
  1589. #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
  1590. extern void clear_huge_page(struct page *page,
  1591. unsigned long addr,
  1592. unsigned int pages_per_huge_page);
  1593. extern void copy_user_huge_page(struct page *dst, struct page *src,
  1594. unsigned long addr, struct vm_area_struct *vma,
  1595. unsigned int pages_per_huge_page);
  1596. #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
  1597. #ifdef CONFIG_DEBUG_PAGEALLOC
  1598. extern unsigned int _debug_guardpage_minorder;
  1599. static inline unsigned int debug_guardpage_minorder(void)
  1600. {
  1601. return _debug_guardpage_minorder;
  1602. }
  1603. static inline bool page_is_guard(struct page *page)
  1604. {
  1605. return test_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  1606. }
  1607. #else
  1608. static inline unsigned int debug_guardpage_minorder(void) { return 0; }
  1609. static inline bool page_is_guard(struct page *page) { return false; }
  1610. #endif /* CONFIG_DEBUG_PAGEALLOC */
  1611. #if MAX_NUMNODES > 1
  1612. void __init setup_nr_node_ids(void);
  1613. #else
  1614. static inline void setup_nr_node_ids(void) {}
  1615. #endif
  1616. #endif /* __KERNEL__ */
  1617. #endif /* _LINUX_MM_H */