mm.h 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315
  1. #ifndef _LINUX_MM_H
  2. #define _LINUX_MM_H
  3. #include <linux/errno.h>
  4. #ifdef __KERNEL__
  5. #include <linux/gfp.h>
  6. #include <linux/list.h>
  7. #include <linux/mmzone.h>
  8. #include <linux/rbtree.h>
  9. #include <linux/prio_tree.h>
  10. #include <linux/debug_locks.h>
  11. #include <linux/mm_types.h>
  12. struct mempolicy;
  13. struct anon_vma;
  14. struct file_ra_state;
  15. struct user_struct;
  16. struct writeback_control;
  17. struct rlimit;
  18. #ifndef CONFIG_DISCONTIGMEM /* Don't use mapnrs, do it properly */
  19. extern unsigned long max_mapnr;
  20. #endif
  21. extern unsigned long num_physpages;
  22. extern void * high_memory;
  23. extern int page_cluster;
  24. #ifdef CONFIG_SYSCTL
  25. extern int sysctl_legacy_va_layout;
  26. #else
  27. #define sysctl_legacy_va_layout 0
  28. #endif
  29. extern unsigned long mmap_min_addr;
  30. #include <asm/page.h>
  31. #include <asm/pgtable.h>
  32. #include <asm/processor.h>
  33. #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
  34. /* to align the pointer to the (next) page boundary */
  35. #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
  36. /*
  37. * Linux kernel virtual memory manager primitives.
  38. * The idea being to have a "virtual" mm in the same way
  39. * we have a virtual fs - giving a cleaner interface to the
  40. * mm details, and allowing different kinds of memory mappings
  41. * (from shared memory to executable loading to arbitrary
  42. * mmap() functions).
  43. */
  44. extern struct kmem_cache *vm_area_cachep;
  45. #ifndef CONFIG_MMU
  46. extern struct rb_root nommu_region_tree;
  47. extern struct rw_semaphore nommu_region_sem;
  48. extern unsigned int kobjsize(const void *objp);
  49. #endif
  50. /*
  51. * vm_flags in vm_area_struct, see mm_types.h.
  52. */
  53. #define VM_READ 0x00000001 /* currently active flags */
  54. #define VM_WRITE 0x00000002
  55. #define VM_EXEC 0x00000004
  56. #define VM_SHARED 0x00000008
  57. /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
  58. #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
  59. #define VM_MAYWRITE 0x00000020
  60. #define VM_MAYEXEC 0x00000040
  61. #define VM_MAYSHARE 0x00000080
  62. #define VM_GROWSDOWN 0x00000100 /* general info on the segment */
  63. #define VM_GROWSUP 0x00000200
  64. #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
  65. #define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
  66. #define VM_EXECUTABLE 0x00001000
  67. #define VM_LOCKED 0x00002000
  68. #define VM_IO 0x00004000 /* Memory mapped I/O or similar */
  69. /* Used by sys_madvise() */
  70. #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
  71. #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
  72. #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
  73. #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
  74. #define VM_RESERVED 0x00080000 /* Count as reserved_vm like IO */
  75. #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
  76. #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
  77. #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
  78. #define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */
  79. #define VM_MAPPED_COPY 0x01000000 /* T if mapped copy of data (nommu mmap) */
  80. #define VM_INSERTPAGE 0x02000000 /* The vma has had "vm_insert_page()" done on it */
  81. #define VM_ALWAYSDUMP 0x04000000 /* Always include in core dumps */
  82. #define VM_CAN_NONLINEAR 0x08000000 /* Has ->fault & does nonlinear pages */
  83. #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
  84. #define VM_SAO 0x20000000 /* Strong Access Ordering (powerpc) */
  85. #define VM_PFN_AT_MMAP 0x40000000 /* PFNMAP vma that is fully mapped at mmap time */
  86. #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
  87. #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
  88. #endif
  89. #ifdef CONFIG_STACK_GROWSUP
  90. #define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  91. #else
  92. #define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
  93. #endif
  94. #define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
  95. #define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
  96. #define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
  97. #define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
  98. #define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
  99. /*
  100. * special vmas that are non-mergable, non-mlock()able
  101. */
  102. #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_RESERVED | VM_PFNMAP)
  103. /*
  104. * mapping from the currently active vm_flags protection bits (the
  105. * low four bits) to a page protection mask..
  106. */
  107. extern pgprot_t protection_map[16];
  108. #define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
  109. #define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */
  110. #define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */
  111. /*
  112. * This interface is used by x86 PAT code to identify a pfn mapping that is
  113. * linear over entire vma. This is to optimize PAT code that deals with
  114. * marking the physical region with a particular prot. This is not for generic
  115. * mm use. Note also that this check will not work if the pfn mapping is
  116. * linear for a vma starting at physical address 0. In which case PAT code
  117. * falls back to slow path of reserving physical range page by page.
  118. */
  119. static inline int is_linear_pfn_mapping(struct vm_area_struct *vma)
  120. {
  121. return (vma->vm_flags & VM_PFN_AT_MMAP);
  122. }
  123. static inline int is_pfn_mapping(struct vm_area_struct *vma)
  124. {
  125. return (vma->vm_flags & VM_PFNMAP);
  126. }
  127. /*
  128. * vm_fault is filled by the the pagefault handler and passed to the vma's
  129. * ->fault function. The vma's ->fault is responsible for returning a bitmask
  130. * of VM_FAULT_xxx flags that give details about how the fault was handled.
  131. *
  132. * pgoff should be used in favour of virtual_address, if possible. If pgoff
  133. * is used, one may set VM_CAN_NONLINEAR in the vma->vm_flags to get nonlinear
  134. * mapping support.
  135. */
  136. struct vm_fault {
  137. unsigned int flags; /* FAULT_FLAG_xxx flags */
  138. pgoff_t pgoff; /* Logical page offset based on vma */
  139. void __user *virtual_address; /* Faulting virtual address */
  140. struct page *page; /* ->fault handlers should return a
  141. * page here, unless VM_FAULT_NOPAGE
  142. * is set (which is also implied by
  143. * VM_FAULT_ERROR).
  144. */
  145. };
  146. /*
  147. * These are the virtual MM functions - opening of an area, closing and
  148. * unmapping it (needed to keep files on disk up-to-date etc), pointer
  149. * to the functions called when a no-page or a wp-page exception occurs.
  150. */
  151. struct vm_operations_struct {
  152. void (*open)(struct vm_area_struct * area);
  153. void (*close)(struct vm_area_struct * area);
  154. int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
  155. /* notification that a previously read-only page is about to become
  156. * writable, if an error is returned it will cause a SIGBUS */
  157. int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
  158. /* called by access_process_vm when get_user_pages() fails, typically
  159. * for use by special VMAs that can switch between memory and hardware
  160. */
  161. int (*access)(struct vm_area_struct *vma, unsigned long addr,
  162. void *buf, int len, int write);
  163. #ifdef CONFIG_NUMA
  164. /*
  165. * set_policy() op must add a reference to any non-NULL @new mempolicy
  166. * to hold the policy upon return. Caller should pass NULL @new to
  167. * remove a policy and fall back to surrounding context--i.e. do not
  168. * install a MPOL_DEFAULT policy, nor the task or system default
  169. * mempolicy.
  170. */
  171. int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
  172. /*
  173. * get_policy() op must add reference [mpol_get()] to any policy at
  174. * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
  175. * in mm/mempolicy.c will do this automatically.
  176. * get_policy() must NOT add a ref if the policy at (vma,addr) is not
  177. * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
  178. * If no [shared/vma] mempolicy exists at the addr, get_policy() op
  179. * must return NULL--i.e., do not "fallback" to task or system default
  180. * policy.
  181. */
  182. struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
  183. unsigned long addr);
  184. int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
  185. const nodemask_t *to, unsigned long flags);
  186. #endif
  187. };
  188. struct mmu_gather;
  189. struct inode;
  190. #define page_private(page) ((page)->private)
  191. #define set_page_private(page, v) ((page)->private = (v))
  192. /*
  193. * FIXME: take this include out, include page-flags.h in
  194. * files which need it (119 of them)
  195. */
  196. #include <linux/page-flags.h>
  197. /*
  198. * Methods to modify the page usage count.
  199. *
  200. * What counts for a page usage:
  201. * - cache mapping (page->mapping)
  202. * - private data (page->private)
  203. * - page mapped in a task's page tables, each mapping
  204. * is counted separately
  205. *
  206. * Also, many kernel routines increase the page count before a critical
  207. * routine so they can be sure the page doesn't go away from under them.
  208. */
  209. /*
  210. * Drop a ref, return true if the refcount fell to zero (the page has no users)
  211. */
  212. static inline int put_page_testzero(struct page *page)
  213. {
  214. VM_BUG_ON(atomic_read(&page->_count) == 0);
  215. return atomic_dec_and_test(&page->_count);
  216. }
  217. /*
  218. * Try to grab a ref unless the page has a refcount of zero, return false if
  219. * that is the case.
  220. */
  221. static inline int get_page_unless_zero(struct page *page)
  222. {
  223. return atomic_inc_not_zero(&page->_count);
  224. }
  225. /* Support for virtually mapped pages */
  226. struct page *vmalloc_to_page(const void *addr);
  227. unsigned long vmalloc_to_pfn(const void *addr);
  228. /*
  229. * Determine if an address is within the vmalloc range
  230. *
  231. * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
  232. * is no special casing required.
  233. */
  234. static inline int is_vmalloc_addr(const void *x)
  235. {
  236. #ifdef CONFIG_MMU
  237. unsigned long addr = (unsigned long)x;
  238. return addr >= VMALLOC_START && addr < VMALLOC_END;
  239. #else
  240. return 0;
  241. #endif
  242. }
  243. static inline struct page *compound_head(struct page *page)
  244. {
  245. if (unlikely(PageTail(page)))
  246. return page->first_page;
  247. return page;
  248. }
  249. static inline int page_count(struct page *page)
  250. {
  251. return atomic_read(&compound_head(page)->_count);
  252. }
  253. static inline void get_page(struct page *page)
  254. {
  255. page = compound_head(page);
  256. VM_BUG_ON(atomic_read(&page->_count) == 0);
  257. atomic_inc(&page->_count);
  258. }
  259. static inline struct page *virt_to_head_page(const void *x)
  260. {
  261. struct page *page = virt_to_page(x);
  262. return compound_head(page);
  263. }
  264. /*
  265. * Setup the page count before being freed into the page allocator for
  266. * the first time (boot or memory hotplug)
  267. */
  268. static inline void init_page_count(struct page *page)
  269. {
  270. atomic_set(&page->_count, 1);
  271. }
  272. void put_page(struct page *page);
  273. void put_pages_list(struct list_head *pages);
  274. void split_page(struct page *page, unsigned int order);
  275. /*
  276. * Compound pages have a destructor function. Provide a
  277. * prototype for that function and accessor functions.
  278. * These are _only_ valid on the head of a PG_compound page.
  279. */
  280. typedef void compound_page_dtor(struct page *);
  281. static inline void set_compound_page_dtor(struct page *page,
  282. compound_page_dtor *dtor)
  283. {
  284. page[1].lru.next = (void *)dtor;
  285. }
  286. static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
  287. {
  288. return (compound_page_dtor *)page[1].lru.next;
  289. }
  290. static inline int compound_order(struct page *page)
  291. {
  292. if (!PageHead(page))
  293. return 0;
  294. return (unsigned long)page[1].lru.prev;
  295. }
  296. static inline void set_compound_order(struct page *page, unsigned long order)
  297. {
  298. page[1].lru.prev = (void *)order;
  299. }
  300. /*
  301. * Multiple processes may "see" the same page. E.g. for untouched
  302. * mappings of /dev/null, all processes see the same page full of
  303. * zeroes, and text pages of executables and shared libraries have
  304. * only one copy in memory, at most, normally.
  305. *
  306. * For the non-reserved pages, page_count(page) denotes a reference count.
  307. * page_count() == 0 means the page is free. page->lru is then used for
  308. * freelist management in the buddy allocator.
  309. * page_count() > 0 means the page has been allocated.
  310. *
  311. * Pages are allocated by the slab allocator in order to provide memory
  312. * to kmalloc and kmem_cache_alloc. In this case, the management of the
  313. * page, and the fields in 'struct page' are the responsibility of mm/slab.c
  314. * unless a particular usage is carefully commented. (the responsibility of
  315. * freeing the kmalloc memory is the caller's, of course).
  316. *
  317. * A page may be used by anyone else who does a __get_free_page().
  318. * In this case, page_count still tracks the references, and should only
  319. * be used through the normal accessor functions. The top bits of page->flags
  320. * and page->virtual store page management information, but all other fields
  321. * are unused and could be used privately, carefully. The management of this
  322. * page is the responsibility of the one who allocated it, and those who have
  323. * subsequently been given references to it.
  324. *
  325. * The other pages (we may call them "pagecache pages") are completely
  326. * managed by the Linux memory manager: I/O, buffers, swapping etc.
  327. * The following discussion applies only to them.
  328. *
  329. * A pagecache page contains an opaque `private' member, which belongs to the
  330. * page's address_space. Usually, this is the address of a circular list of
  331. * the page's disk buffers. PG_private must be set to tell the VM to call
  332. * into the filesystem to release these pages.
  333. *
  334. * A page may belong to an inode's memory mapping. In this case, page->mapping
  335. * is the pointer to the inode, and page->index is the file offset of the page,
  336. * in units of PAGE_CACHE_SIZE.
  337. *
  338. * If pagecache pages are not associated with an inode, they are said to be
  339. * anonymous pages. These may become associated with the swapcache, and in that
  340. * case PG_swapcache is set, and page->private is an offset into the swapcache.
  341. *
  342. * In either case (swapcache or inode backed), the pagecache itself holds one
  343. * reference to the page. Setting PG_private should also increment the
  344. * refcount. The each user mapping also has a reference to the page.
  345. *
  346. * The pagecache pages are stored in a per-mapping radix tree, which is
  347. * rooted at mapping->page_tree, and indexed by offset.
  348. * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
  349. * lists, we instead now tag pages as dirty/writeback in the radix tree.
  350. *
  351. * All pagecache pages may be subject to I/O:
  352. * - inode pages may need to be read from disk,
  353. * - inode pages which have been modified and are MAP_SHARED may need
  354. * to be written back to the inode on disk,
  355. * - anonymous pages (including MAP_PRIVATE file mappings) which have been
  356. * modified may need to be swapped out to swap space and (later) to be read
  357. * back into memory.
  358. */
  359. /*
  360. * The zone field is never updated after free_area_init_core()
  361. * sets it, so none of the operations on it need to be atomic.
  362. */
  363. /*
  364. * page->flags layout:
  365. *
  366. * There are three possibilities for how page->flags get
  367. * laid out. The first is for the normal case, without
  368. * sparsemem. The second is for sparsemem when there is
  369. * plenty of space for node and section. The last is when
  370. * we have run out of space and have to fall back to an
  371. * alternate (slower) way of determining the node.
  372. *
  373. * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
  374. * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
  375. * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
  376. */
  377. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  378. #define SECTIONS_WIDTH SECTIONS_SHIFT
  379. #else
  380. #define SECTIONS_WIDTH 0
  381. #endif
  382. #define ZONES_WIDTH ZONES_SHIFT
  383. #if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
  384. #define NODES_WIDTH NODES_SHIFT
  385. #else
  386. #ifdef CONFIG_SPARSEMEM_VMEMMAP
  387. #error "Vmemmap: No space for nodes field in page flags"
  388. #endif
  389. #define NODES_WIDTH 0
  390. #endif
  391. /* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */
  392. #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
  393. #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
  394. #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
  395. /*
  396. * We are going to use the flags for the page to node mapping if its in
  397. * there. This includes the case where there is no node, so it is implicit.
  398. */
  399. #if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
  400. #define NODE_NOT_IN_PAGE_FLAGS
  401. #endif
  402. #ifndef PFN_SECTION_SHIFT
  403. #define PFN_SECTION_SHIFT 0
  404. #endif
  405. /*
  406. * Define the bit shifts to access each section. For non-existant
  407. * sections we define the shift as 0; that plus a 0 mask ensures
  408. * the compiler will optimise away reference to them.
  409. */
  410. #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
  411. #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
  412. #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
  413. /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allcator */
  414. #ifdef NODE_NOT_IN_PAGEFLAGS
  415. #define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
  416. #define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
  417. SECTIONS_PGOFF : ZONES_PGOFF)
  418. #else
  419. #define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
  420. #define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
  421. NODES_PGOFF : ZONES_PGOFF)
  422. #endif
  423. #define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
  424. #if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  425. #error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
  426. #endif
  427. #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
  428. #define NODES_MASK ((1UL << NODES_WIDTH) - 1)
  429. #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
  430. #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
  431. static inline enum zone_type page_zonenum(struct page *page)
  432. {
  433. return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
  434. }
  435. /*
  436. * The identification function is only used by the buddy allocator for
  437. * determining if two pages could be buddies. We are not really
  438. * identifying a zone since we could be using a the section number
  439. * id if we have not node id available in page flags.
  440. * We guarantee only that it will return the same value for two
  441. * combinable pages in a zone.
  442. */
  443. static inline int page_zone_id(struct page *page)
  444. {
  445. return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
  446. }
  447. static inline int zone_to_nid(struct zone *zone)
  448. {
  449. #ifdef CONFIG_NUMA
  450. return zone->node;
  451. #else
  452. return 0;
  453. #endif
  454. }
  455. #ifdef NODE_NOT_IN_PAGE_FLAGS
  456. extern int page_to_nid(struct page *page);
  457. #else
  458. static inline int page_to_nid(struct page *page)
  459. {
  460. return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
  461. }
  462. #endif
  463. static inline struct zone *page_zone(struct page *page)
  464. {
  465. return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
  466. }
  467. #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
  468. static inline unsigned long page_to_section(struct page *page)
  469. {
  470. return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
  471. }
  472. #endif
  473. static inline void set_page_zone(struct page *page, enum zone_type zone)
  474. {
  475. page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
  476. page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
  477. }
  478. static inline void set_page_node(struct page *page, unsigned long node)
  479. {
  480. page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
  481. page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
  482. }
  483. static inline void set_page_section(struct page *page, unsigned long section)
  484. {
  485. page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
  486. page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
  487. }
  488. static inline void set_page_links(struct page *page, enum zone_type zone,
  489. unsigned long node, unsigned long pfn)
  490. {
  491. set_page_zone(page, zone);
  492. set_page_node(page, node);
  493. set_page_section(page, pfn_to_section_nr(pfn));
  494. }
  495. /*
  496. * If a hint addr is less than mmap_min_addr change hint to be as
  497. * low as possible but still greater than mmap_min_addr
  498. */
  499. static inline unsigned long round_hint_to_min(unsigned long hint)
  500. {
  501. hint &= PAGE_MASK;
  502. if (((void *)hint != NULL) &&
  503. (hint < mmap_min_addr))
  504. return PAGE_ALIGN(mmap_min_addr);
  505. return hint;
  506. }
  507. /*
  508. * Some inline functions in vmstat.h depend on page_zone()
  509. */
  510. #include <linux/vmstat.h>
  511. static __always_inline void *lowmem_page_address(struct page *page)
  512. {
  513. return __va(page_to_pfn(page) << PAGE_SHIFT);
  514. }
  515. #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
  516. #define HASHED_PAGE_VIRTUAL
  517. #endif
  518. #if defined(WANT_PAGE_VIRTUAL)
  519. #define page_address(page) ((page)->virtual)
  520. #define set_page_address(page, address) \
  521. do { \
  522. (page)->virtual = (address); \
  523. } while(0)
  524. #define page_address_init() do { } while(0)
  525. #endif
  526. #if defined(HASHED_PAGE_VIRTUAL)
  527. void *page_address(struct page *page);
  528. void set_page_address(struct page *page, void *virtual);
  529. void page_address_init(void);
  530. #endif
  531. #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
  532. #define page_address(page) lowmem_page_address(page)
  533. #define set_page_address(page, address) do { } while(0)
  534. #define page_address_init() do { } while(0)
  535. #endif
  536. /*
  537. * On an anonymous page mapped into a user virtual memory area,
  538. * page->mapping points to its anon_vma, not to a struct address_space;
  539. * with the PAGE_MAPPING_ANON bit set to distinguish it.
  540. *
  541. * Please note that, confusingly, "page_mapping" refers to the inode
  542. * address_space which maps the page from disk; whereas "page_mapped"
  543. * refers to user virtual address space into which the page is mapped.
  544. */
  545. #define PAGE_MAPPING_ANON 1
  546. extern struct address_space swapper_space;
  547. static inline struct address_space *page_mapping(struct page *page)
  548. {
  549. struct address_space *mapping = page->mapping;
  550. VM_BUG_ON(PageSlab(page));
  551. #ifdef CONFIG_SWAP
  552. if (unlikely(PageSwapCache(page)))
  553. mapping = &swapper_space;
  554. else
  555. #endif
  556. if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
  557. mapping = NULL;
  558. return mapping;
  559. }
  560. static inline int PageAnon(struct page *page)
  561. {
  562. return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
  563. }
  564. /*
  565. * Return the pagecache index of the passed page. Regular pagecache pages
  566. * use ->index whereas swapcache pages use ->private
  567. */
  568. static inline pgoff_t page_index(struct page *page)
  569. {
  570. if (unlikely(PageSwapCache(page)))
  571. return page_private(page);
  572. return page->index;
  573. }
  574. /*
  575. * The atomic page->_mapcount, like _count, starts from -1:
  576. * so that transitions both from it and to it can be tracked,
  577. * using atomic_inc_and_test and atomic_add_negative(-1).
  578. */
  579. static inline void reset_page_mapcount(struct page *page)
  580. {
  581. atomic_set(&(page)->_mapcount, -1);
  582. }
  583. static inline int page_mapcount(struct page *page)
  584. {
  585. return atomic_read(&(page)->_mapcount) + 1;
  586. }
  587. /*
  588. * Return true if this page is mapped into pagetables.
  589. */
  590. static inline int page_mapped(struct page *page)
  591. {
  592. return atomic_read(&(page)->_mapcount) >= 0;
  593. }
  594. /*
  595. * Different kinds of faults, as returned by handle_mm_fault().
  596. * Used to decide whether a process gets delivered SIGBUS or
  597. * just gets major/minor fault counters bumped up.
  598. */
  599. #define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
  600. #define VM_FAULT_OOM 0x0001
  601. #define VM_FAULT_SIGBUS 0x0002
  602. #define VM_FAULT_MAJOR 0x0004
  603. #define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
  604. #define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
  605. #define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
  606. #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
  607. /*
  608. * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
  609. */
  610. extern void pagefault_out_of_memory(void);
  611. #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
  612. extern void show_free_areas(void);
  613. #ifdef CONFIG_SHMEM
  614. extern int shmem_lock(struct file *file, int lock, struct user_struct *user);
  615. #else
  616. static inline int shmem_lock(struct file *file, int lock,
  617. struct user_struct *user)
  618. {
  619. return 0;
  620. }
  621. #endif
  622. struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
  623. int shmem_zero_setup(struct vm_area_struct *);
  624. #ifndef CONFIG_MMU
  625. extern unsigned long shmem_get_unmapped_area(struct file *file,
  626. unsigned long addr,
  627. unsigned long len,
  628. unsigned long pgoff,
  629. unsigned long flags);
  630. #endif
  631. extern int can_do_mlock(void);
  632. extern int user_shm_lock(size_t, struct user_struct *);
  633. extern void user_shm_unlock(size_t, struct user_struct *);
  634. /*
  635. * Parameter block passed down to zap_pte_range in exceptional cases.
  636. */
  637. struct zap_details {
  638. struct vm_area_struct *nonlinear_vma; /* Check page->index if set */
  639. struct address_space *check_mapping; /* Check page->mapping if set */
  640. pgoff_t first_index; /* Lowest page->index to unmap */
  641. pgoff_t last_index; /* Highest page->index to unmap */
  642. spinlock_t *i_mmap_lock; /* For unmap_mapping_range: */
  643. unsigned long truncate_count; /* Compare vm_truncate_count */
  644. };
  645. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  646. pte_t pte);
  647. int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  648. unsigned long size);
  649. unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
  650. unsigned long size, struct zap_details *);
  651. unsigned long unmap_vmas(struct mmu_gather **tlb,
  652. struct vm_area_struct *start_vma, unsigned long start_addr,
  653. unsigned long end_addr, unsigned long *nr_accounted,
  654. struct zap_details *);
  655. /**
  656. * mm_walk - callbacks for walk_page_range
  657. * @pgd_entry: if set, called for each non-empty PGD (top-level) entry
  658. * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry
  659. * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
  660. * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
  661. * @pte_hole: if set, called for each hole at all levels
  662. *
  663. * (see walk_page_range for more details)
  664. */
  665. struct mm_walk {
  666. int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
  667. int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
  668. int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
  669. int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
  670. int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
  671. struct mm_struct *mm;
  672. void *private;
  673. };
  674. int walk_page_range(unsigned long addr, unsigned long end,
  675. struct mm_walk *walk);
  676. void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
  677. unsigned long end, unsigned long floor, unsigned long ceiling);
  678. int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
  679. struct vm_area_struct *vma);
  680. void unmap_mapping_range(struct address_space *mapping,
  681. loff_t const holebegin, loff_t const holelen, int even_cows);
  682. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  683. unsigned long *pfn);
  684. int follow_phys(struct vm_area_struct *vma, unsigned long address,
  685. unsigned int flags, unsigned long *prot, resource_size_t *phys);
  686. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  687. void *buf, int len, int write);
  688. static inline void unmap_shared_mapping_range(struct address_space *mapping,
  689. loff_t const holebegin, loff_t const holelen)
  690. {
  691. unmap_mapping_range(mapping, holebegin, holelen, 0);
  692. }
  693. extern int vmtruncate(struct inode * inode, loff_t offset);
  694. extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
  695. #ifdef CONFIG_MMU
  696. extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  697. unsigned long address, int write_access);
  698. #else
  699. static inline int handle_mm_fault(struct mm_struct *mm,
  700. struct vm_area_struct *vma, unsigned long address,
  701. int write_access)
  702. {
  703. /* should never happen if there's no MMU */
  704. BUG();
  705. return VM_FAULT_SIGBUS;
  706. }
  707. #endif
  708. extern int make_pages_present(unsigned long addr, unsigned long end);
  709. extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
  710. int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  711. unsigned long start, int len, int write, int force,
  712. struct page **pages, struct vm_area_struct **vmas);
  713. int get_user_pages_fast(unsigned long start, int nr_pages, int write,
  714. struct page **pages);
  715. extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
  716. extern void do_invalidatepage(struct page *page, unsigned long offset);
  717. int __set_page_dirty_nobuffers(struct page *page);
  718. int __set_page_dirty_no_writeback(struct page *page);
  719. int redirty_page_for_writepage(struct writeback_control *wbc,
  720. struct page *page);
  721. void account_page_dirtied(struct page *page, struct address_space *mapping);
  722. int set_page_dirty(struct page *page);
  723. int set_page_dirty_lock(struct page *page);
  724. int clear_page_dirty_for_io(struct page *page);
  725. extern unsigned long move_page_tables(struct vm_area_struct *vma,
  726. unsigned long old_addr, struct vm_area_struct *new_vma,
  727. unsigned long new_addr, unsigned long len);
  728. extern unsigned long do_mremap(unsigned long addr,
  729. unsigned long old_len, unsigned long new_len,
  730. unsigned long flags, unsigned long new_addr);
  731. extern int mprotect_fixup(struct vm_area_struct *vma,
  732. struct vm_area_struct **pprev, unsigned long start,
  733. unsigned long end, unsigned long newflags);
  734. /*
  735. * A callback you can register to apply pressure to ageable caches.
  736. *
  737. * 'shrink' is passed a count 'nr_to_scan' and a 'gfpmask'. It should
  738. * look through the least-recently-used 'nr_to_scan' entries and
  739. * attempt to free them up. It should return the number of objects
  740. * which remain in the cache. If it returns -1, it means it cannot do
  741. * any scanning at this time (eg. there is a risk of deadlock).
  742. *
  743. * The 'gfpmask' refers to the allocation we are currently trying to
  744. * fulfil.
  745. *
  746. * Note that 'shrink' will be passed nr_to_scan == 0 when the VM is
  747. * querying the cache size, so a fastpath for that case is appropriate.
  748. */
  749. struct shrinker {
  750. int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
  751. int seeks; /* seeks to recreate an obj */
  752. /* These are for internal use */
  753. struct list_head list;
  754. long nr; /* objs pending delete */
  755. };
  756. #define DEFAULT_SEEKS 2 /* A good number if you don't know better. */
  757. extern void register_shrinker(struct shrinker *);
  758. extern void unregister_shrinker(struct shrinker *);
  759. int vma_wants_writenotify(struct vm_area_struct *vma);
  760. extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
  761. #ifdef __PAGETABLE_PUD_FOLDED
  762. static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
  763. unsigned long address)
  764. {
  765. return 0;
  766. }
  767. #else
  768. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
  769. #endif
  770. #ifdef __PAGETABLE_PMD_FOLDED
  771. static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
  772. unsigned long address)
  773. {
  774. return 0;
  775. }
  776. #else
  777. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
  778. #endif
  779. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
  780. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
  781. /*
  782. * The following ifdef needed to get the 4level-fixup.h header to work.
  783. * Remove it when 4level-fixup.h has been removed.
  784. */
  785. #if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
  786. static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  787. {
  788. return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
  789. NULL: pud_offset(pgd, address);
  790. }
  791. static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  792. {
  793. return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
  794. NULL: pmd_offset(pud, address);
  795. }
  796. #endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
  797. #if USE_SPLIT_PTLOCKS
  798. /*
  799. * We tuck a spinlock to guard each pagetable page into its struct page,
  800. * at page->private, with BUILD_BUG_ON to make sure that this will not
  801. * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
  802. * When freeing, reset page->mapping so free_pages_check won't complain.
  803. */
  804. #define __pte_lockptr(page) &((page)->ptl)
  805. #define pte_lock_init(_page) do { \
  806. spin_lock_init(__pte_lockptr(_page)); \
  807. } while (0)
  808. #define pte_lock_deinit(page) ((page)->mapping = NULL)
  809. #define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
  810. #else /* !USE_SPLIT_PTLOCKS */
  811. /*
  812. * We use mm->page_table_lock to guard all pagetable pages of the mm.
  813. */
  814. #define pte_lock_init(page) do {} while (0)
  815. #define pte_lock_deinit(page) do {} while (0)
  816. #define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
  817. #endif /* USE_SPLIT_PTLOCKS */
  818. static inline void pgtable_page_ctor(struct page *page)
  819. {
  820. pte_lock_init(page);
  821. inc_zone_page_state(page, NR_PAGETABLE);
  822. }
  823. static inline void pgtable_page_dtor(struct page *page)
  824. {
  825. pte_lock_deinit(page);
  826. dec_zone_page_state(page, NR_PAGETABLE);
  827. }
  828. #define pte_offset_map_lock(mm, pmd, address, ptlp) \
  829. ({ \
  830. spinlock_t *__ptl = pte_lockptr(mm, pmd); \
  831. pte_t *__pte = pte_offset_map(pmd, address); \
  832. *(ptlp) = __ptl; \
  833. spin_lock(__ptl); \
  834. __pte; \
  835. })
  836. #define pte_unmap_unlock(pte, ptl) do { \
  837. spin_unlock(ptl); \
  838. pte_unmap(pte); \
  839. } while (0)
  840. #define pte_alloc_map(mm, pmd, address) \
  841. ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
  842. NULL: pte_offset_map(pmd, address))
  843. #define pte_alloc_map_lock(mm, pmd, address, ptlp) \
  844. ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
  845. NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
  846. #define pte_alloc_kernel(pmd, address) \
  847. ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
  848. NULL: pte_offset_kernel(pmd, address))
  849. extern void free_area_init(unsigned long * zones_size);
  850. extern void free_area_init_node(int nid, unsigned long * zones_size,
  851. unsigned long zone_start_pfn, unsigned long *zholes_size);
  852. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  853. /*
  854. * With CONFIG_ARCH_POPULATES_NODE_MAP set, an architecture may initialise its
  855. * zones, allocate the backing mem_map and account for memory holes in a more
  856. * architecture independent manner. This is a substitute for creating the
  857. * zone_sizes[] and zholes_size[] arrays and passing them to
  858. * free_area_init_node()
  859. *
  860. * An architecture is expected to register range of page frames backed by
  861. * physical memory with add_active_range() before calling
  862. * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
  863. * usage, an architecture is expected to do something like
  864. *
  865. * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
  866. * max_highmem_pfn};
  867. * for_each_valid_physical_page_range()
  868. * add_active_range(node_id, start_pfn, end_pfn)
  869. * free_area_init_nodes(max_zone_pfns);
  870. *
  871. * If the architecture guarantees that there are no holes in the ranges
  872. * registered with add_active_range(), free_bootmem_active_regions()
  873. * will call free_bootmem_node() for each registered physical page range.
  874. * Similarly sparse_memory_present_with_active_regions() calls
  875. * memory_present() for each range when SPARSEMEM is enabled.
  876. *
  877. * See mm/page_alloc.c for more information on each function exposed by
  878. * CONFIG_ARCH_POPULATES_NODE_MAP
  879. */
  880. extern void free_area_init_nodes(unsigned long *max_zone_pfn);
  881. extern void add_active_range(unsigned int nid, unsigned long start_pfn,
  882. unsigned long end_pfn);
  883. extern void remove_active_range(unsigned int nid, unsigned long start_pfn,
  884. unsigned long end_pfn);
  885. extern void remove_all_active_ranges(void);
  886. extern unsigned long absent_pages_in_range(unsigned long start_pfn,
  887. unsigned long end_pfn);
  888. extern void get_pfn_range_for_nid(unsigned int nid,
  889. unsigned long *start_pfn, unsigned long *end_pfn);
  890. extern unsigned long find_min_pfn_with_active_regions(void);
  891. extern void free_bootmem_with_active_regions(int nid,
  892. unsigned long max_low_pfn);
  893. typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
  894. extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data);
  895. extern void sparse_memory_present_with_active_regions(int nid);
  896. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  897. #if !defined(CONFIG_ARCH_POPULATES_NODE_MAP) && \
  898. !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
  899. static inline int __early_pfn_to_nid(unsigned long pfn)
  900. {
  901. return 0;
  902. }
  903. #else
  904. /* please see mm/page_alloc.c */
  905. extern int __meminit early_pfn_to_nid(unsigned long pfn);
  906. #ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  907. /* there is a per-arch backend function. */
  908. extern int __meminit __early_pfn_to_nid(unsigned long pfn);
  909. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  910. #endif
  911. extern void set_dma_reserve(unsigned long new_dma_reserve);
  912. extern void memmap_init_zone(unsigned long, int, unsigned long,
  913. unsigned long, enum memmap_context);
  914. extern void setup_per_zone_pages_min(void);
  915. extern void mem_init(void);
  916. extern void __init mmap_init(void);
  917. extern void show_mem(void);
  918. extern void si_meminfo(struct sysinfo * val);
  919. extern void si_meminfo_node(struct sysinfo *val, int nid);
  920. extern int after_bootmem;
  921. #ifdef CONFIG_NUMA
  922. extern void setup_per_cpu_pageset(void);
  923. #else
  924. static inline void setup_per_cpu_pageset(void) {}
  925. #endif
  926. /* nommu.c */
  927. extern atomic_long_t mmap_pages_allocated;
  928. /* prio_tree.c */
  929. void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
  930. void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
  931. void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
  932. struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
  933. struct prio_tree_iter *iter);
  934. #define vma_prio_tree_foreach(vma, iter, root, begin, end) \
  935. for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
  936. (vma = vma_prio_tree_next(vma, iter)); )
  937. static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
  938. struct list_head *list)
  939. {
  940. vma->shared.vm_set.parent = NULL;
  941. list_add_tail(&vma->shared.vm_set.list, list);
  942. }
  943. /* mmap.c */
  944. extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
  945. extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
  946. unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
  947. extern struct vm_area_struct *vma_merge(struct mm_struct *,
  948. struct vm_area_struct *prev, unsigned long addr, unsigned long end,
  949. unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
  950. struct mempolicy *);
  951. extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
  952. extern int split_vma(struct mm_struct *,
  953. struct vm_area_struct *, unsigned long addr, int new_below);
  954. extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
  955. extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
  956. struct rb_node **, struct rb_node *);
  957. extern void unlink_file_vma(struct vm_area_struct *);
  958. extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
  959. unsigned long addr, unsigned long len, pgoff_t pgoff);
  960. extern void exit_mmap(struct mm_struct *);
  961. extern int mm_take_all_locks(struct mm_struct *mm);
  962. extern void mm_drop_all_locks(struct mm_struct *mm);
  963. #ifdef CONFIG_PROC_FS
  964. /* From fs/proc/base.c. callers must _not_ hold the mm's exe_file_lock */
  965. extern void added_exe_file_vma(struct mm_struct *mm);
  966. extern void removed_exe_file_vma(struct mm_struct *mm);
  967. #else
  968. static inline void added_exe_file_vma(struct mm_struct *mm)
  969. {}
  970. static inline void removed_exe_file_vma(struct mm_struct *mm)
  971. {}
  972. #endif /* CONFIG_PROC_FS */
  973. extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
  974. extern int install_special_mapping(struct mm_struct *mm,
  975. unsigned long addr, unsigned long len,
  976. unsigned long flags, struct page **pages);
  977. extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
  978. extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
  979. unsigned long len, unsigned long prot,
  980. unsigned long flag, unsigned long pgoff);
  981. extern unsigned long mmap_region(struct file *file, unsigned long addr,
  982. unsigned long len, unsigned long flags,
  983. unsigned int vm_flags, unsigned long pgoff);
  984. static inline unsigned long do_mmap(struct file *file, unsigned long addr,
  985. unsigned long len, unsigned long prot,
  986. unsigned long flag, unsigned long offset)
  987. {
  988. unsigned long ret = -EINVAL;
  989. if ((offset + PAGE_ALIGN(len)) < offset)
  990. goto out;
  991. if (!(offset & ~PAGE_MASK))
  992. ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
  993. out:
  994. return ret;
  995. }
  996. extern int do_munmap(struct mm_struct *, unsigned long, size_t);
  997. extern unsigned long do_brk(unsigned long, unsigned long);
  998. /* filemap.c */
  999. extern unsigned long page_unuse(struct page *);
  1000. extern void truncate_inode_pages(struct address_space *, loff_t);
  1001. extern void truncate_inode_pages_range(struct address_space *,
  1002. loff_t lstart, loff_t lend);
  1003. /* generic vm_area_ops exported for stackable file systems */
  1004. extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
  1005. /* mm/page-writeback.c */
  1006. int write_one_page(struct page *page, int wait);
  1007. void task_dirty_inc(struct task_struct *tsk);
  1008. /* readahead.c */
  1009. #define VM_MAX_READAHEAD 128 /* kbytes */
  1010. #define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
  1011. int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
  1012. pgoff_t offset, unsigned long nr_to_read);
  1013. void page_cache_sync_readahead(struct address_space *mapping,
  1014. struct file_ra_state *ra,
  1015. struct file *filp,
  1016. pgoff_t offset,
  1017. unsigned long size);
  1018. void page_cache_async_readahead(struct address_space *mapping,
  1019. struct file_ra_state *ra,
  1020. struct file *filp,
  1021. struct page *pg,
  1022. pgoff_t offset,
  1023. unsigned long size);
  1024. unsigned long max_sane_readahead(unsigned long nr);
  1025. unsigned long ra_submit(struct file_ra_state *ra,
  1026. struct address_space *mapping,
  1027. struct file *filp);
  1028. /* Do stack extension */
  1029. extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
  1030. #ifdef CONFIG_IA64
  1031. extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
  1032. #endif
  1033. extern int expand_stack_downwards(struct vm_area_struct *vma,
  1034. unsigned long address);
  1035. /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
  1036. extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
  1037. extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
  1038. struct vm_area_struct **pprev);
  1039. /* Look up the first VMA which intersects the interval start_addr..end_addr-1,
  1040. NULL if none. Assume start_addr < end_addr. */
  1041. static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
  1042. {
  1043. struct vm_area_struct * vma = find_vma(mm,start_addr);
  1044. if (vma && end_addr <= vma->vm_start)
  1045. vma = NULL;
  1046. return vma;
  1047. }
  1048. static inline unsigned long vma_pages(struct vm_area_struct *vma)
  1049. {
  1050. return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  1051. }
  1052. pgprot_t vm_get_page_prot(unsigned long vm_flags);
  1053. struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
  1054. int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
  1055. unsigned long pfn, unsigned long size, pgprot_t);
  1056. int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
  1057. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1058. unsigned long pfn);
  1059. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1060. unsigned long pfn);
  1061. struct page *follow_page(struct vm_area_struct *, unsigned long address,
  1062. unsigned int foll_flags);
  1063. #define FOLL_WRITE 0x01 /* check pte is writable */
  1064. #define FOLL_TOUCH 0x02 /* mark page accessed */
  1065. #define FOLL_GET 0x04 /* do get_page on page */
  1066. #define FOLL_ANON 0x08 /* give ZERO_PAGE if no pgtable */
  1067. typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
  1068. void *data);
  1069. extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
  1070. unsigned long size, pte_fn_t fn, void *data);
  1071. #ifdef CONFIG_PROC_FS
  1072. void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
  1073. #else
  1074. static inline void vm_stat_account(struct mm_struct *mm,
  1075. unsigned long flags, struct file *file, long pages)
  1076. {
  1077. }
  1078. #endif /* CONFIG_PROC_FS */
  1079. #ifdef CONFIG_DEBUG_PAGEALLOC
  1080. extern int debug_pagealloc_enabled;
  1081. extern void kernel_map_pages(struct page *page, int numpages, int enable);
  1082. static inline void enable_debug_pagealloc(void)
  1083. {
  1084. debug_pagealloc_enabled = 1;
  1085. }
  1086. #ifdef CONFIG_HIBERNATION
  1087. extern bool kernel_page_present(struct page *page);
  1088. #endif /* CONFIG_HIBERNATION */
  1089. #else
  1090. static inline void
  1091. kernel_map_pages(struct page *page, int numpages, int enable) {}
  1092. static inline void enable_debug_pagealloc(void)
  1093. {
  1094. }
  1095. #ifdef CONFIG_HIBERNATION
  1096. static inline bool kernel_page_present(struct page *page) { return true; }
  1097. #endif /* CONFIG_HIBERNATION */
  1098. #endif
  1099. extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
  1100. #ifdef __HAVE_ARCH_GATE_AREA
  1101. int in_gate_area_no_task(unsigned long addr);
  1102. int in_gate_area(struct task_struct *task, unsigned long addr);
  1103. #else
  1104. int in_gate_area_no_task(unsigned long addr);
  1105. #define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
  1106. #endif /* __HAVE_ARCH_GATE_AREA */
  1107. int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
  1108. void __user *, size_t *, loff_t *);
  1109. unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
  1110. unsigned long lru_pages);
  1111. #ifndef CONFIG_MMU
  1112. #define randomize_va_space 0
  1113. #else
  1114. extern int randomize_va_space;
  1115. #endif
  1116. const char * arch_vma_name(struct vm_area_struct *vma);
  1117. void print_vma_addr(char *prefix, unsigned long rip);
  1118. struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
  1119. pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
  1120. pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
  1121. pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
  1122. pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
  1123. void *vmemmap_alloc_block(unsigned long size, int node);
  1124. void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
  1125. int vmemmap_populate_basepages(struct page *start_page,
  1126. unsigned long pages, int node);
  1127. int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
  1128. void vmemmap_populate_print_last(void);
  1129. extern int account_locked_memory(struct mm_struct *mm, struct rlimit *rlim,
  1130. size_t size);
  1131. extern void refund_locked_memory(struct mm_struct *mm, size_t size);
  1132. #endif /* __KERNEL__ */
  1133. #endif /* _LINUX_MM_H */