mmzone.h 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259
  1. #ifndef _LINUX_MMZONE_H
  2. #define _LINUX_MMZONE_H
  3. #ifndef __ASSEMBLY__
  4. #ifndef __GENERATING_BOUNDS_H
  5. #include <linux/spinlock.h>
  6. #include <linux/list.h>
  7. #include <linux/wait.h>
  8. #include <linux/bitops.h>
  9. #include <linux/cache.h>
  10. #include <linux/threads.h>
  11. #include <linux/numa.h>
  12. #include <linux/init.h>
  13. #include <linux/seqlock.h>
  14. #include <linux/nodemask.h>
  15. #include <linux/pageblock-flags.h>
  16. #include <linux/page-flags-layout.h>
  17. #include <linux/atomic.h>
  18. #include <asm/page.h>
  19. /* Free memory management - zoned buddy allocator. */
  20. #ifndef CONFIG_FORCE_MAX_ZONEORDER
  21. #define MAX_ORDER 11
  22. #else
  23. #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
  24. #endif
  25. #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
  26. /*
  27. * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
  28. * costly to service. That is between allocation orders which should
  29. * coalesce naturally under reasonable reclaim pressure and those which
  30. * will not.
  31. */
  32. #define PAGE_ALLOC_COSTLY_ORDER 3
  33. enum {
  34. MIGRATE_UNMOVABLE,
  35. MIGRATE_RECLAIMABLE,
  36. MIGRATE_MOVABLE,
  37. MIGRATE_PCPTYPES, /* the number of types on the pcp lists */
  38. MIGRATE_RESERVE = MIGRATE_PCPTYPES,
  39. #ifdef CONFIG_CMA
  40. /*
  41. * MIGRATE_CMA migration type is designed to mimic the way
  42. * ZONE_MOVABLE works. Only movable pages can be allocated
  43. * from MIGRATE_CMA pageblocks and page allocator never
  44. * implicitly change migration type of MIGRATE_CMA pageblock.
  45. *
  46. * The way to use it is to change migratetype of a range of
  47. * pageblocks to MIGRATE_CMA which can be done by
  48. * __free_pageblock_cma() function. What is important though
  49. * is that a range of pageblocks must be aligned to
  50. * MAX_ORDER_NR_PAGES should biggest page be bigger then
  51. * a single pageblock.
  52. */
  53. MIGRATE_CMA,
  54. #endif
  55. #ifdef CONFIG_MEMORY_ISOLATION
  56. MIGRATE_ISOLATE, /* can't allocate from here */
  57. #endif
  58. MIGRATE_TYPES
  59. };
  60. #ifdef CONFIG_CMA
  61. # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
  62. #else
  63. # define is_migrate_cma(migratetype) false
  64. #endif
  65. #define for_each_migratetype_order(order, type) \
  66. for (order = 0; order < MAX_ORDER; order++) \
  67. for (type = 0; type < MIGRATE_TYPES; type++)
  68. extern int page_group_by_mobility_disabled;
  69. static inline int get_pageblock_migratetype(struct page *page)
  70. {
  71. return get_pageblock_flags_group(page, PB_migrate, PB_migrate_end);
  72. }
  73. struct free_area {
  74. struct list_head free_list[MIGRATE_TYPES];
  75. unsigned long nr_free;
  76. };
  77. struct pglist_data;
  78. /*
  79. * zone->lock and zone->lru_lock are two of the hottest locks in the kernel.
  80. * So add a wild amount of padding here to ensure that they fall into separate
  81. * cachelines. There are very few zone structures in the machine, so space
  82. * consumption is not a concern here.
  83. */
  84. #if defined(CONFIG_SMP)
  85. struct zone_padding {
  86. char x[0];
  87. } ____cacheline_internodealigned_in_smp;
  88. #define ZONE_PADDING(name) struct zone_padding name;
  89. #else
  90. #define ZONE_PADDING(name)
  91. #endif
  92. enum zone_stat_item {
  93. /* First 128 byte cacheline (assuming 64 bit words) */
  94. NR_FREE_PAGES,
  95. NR_LRU_BASE,
  96. NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
  97. NR_ACTIVE_ANON, /* " " " " " */
  98. NR_INACTIVE_FILE, /* " " " " " */
  99. NR_ACTIVE_FILE, /* " " " " " */
  100. NR_UNEVICTABLE, /* " " " " " */
  101. NR_MLOCK, /* mlock()ed pages found and moved off LRU */
  102. NR_ANON_PAGES, /* Mapped anonymous pages */
  103. NR_FILE_MAPPED, /* pagecache pages mapped into pagetables.
  104. only modified from process context */
  105. NR_FILE_PAGES,
  106. NR_FILE_DIRTY,
  107. NR_WRITEBACK,
  108. NR_SLAB_RECLAIMABLE,
  109. NR_SLAB_UNRECLAIMABLE,
  110. NR_PAGETABLE, /* used for pagetables */
  111. NR_KERNEL_STACK,
  112. /* Second 128 byte cacheline */
  113. NR_UNSTABLE_NFS, /* NFS unstable pages */
  114. NR_BOUNCE,
  115. NR_VMSCAN_WRITE,
  116. NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */
  117. NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */
  118. NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */
  119. NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */
  120. NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */
  121. NR_DIRTIED, /* page dirtyings since bootup */
  122. NR_WRITTEN, /* page writings since bootup */
  123. #ifdef CONFIG_NUMA
  124. NUMA_HIT, /* allocated in intended node */
  125. NUMA_MISS, /* allocated in non intended node */
  126. NUMA_FOREIGN, /* was intended here, hit elsewhere */
  127. NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
  128. NUMA_LOCAL, /* allocation from local node */
  129. NUMA_OTHER, /* allocation from other node */
  130. #endif
  131. NR_ANON_TRANSPARENT_HUGEPAGES,
  132. NR_FREE_CMA_PAGES,
  133. NR_VM_ZONE_STAT_ITEMS };
  134. /*
  135. * We do arithmetic on the LRU lists in various places in the code,
  136. * so it is important to keep the active lists LRU_ACTIVE higher in
  137. * the array than the corresponding inactive lists, and to keep
  138. * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
  139. *
  140. * This has to be kept in sync with the statistics in zone_stat_item
  141. * above and the descriptions in vmstat_text in mm/vmstat.c
  142. */
  143. #define LRU_BASE 0
  144. #define LRU_ACTIVE 1
  145. #define LRU_FILE 2
  146. enum lru_list {
  147. LRU_INACTIVE_ANON = LRU_BASE,
  148. LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
  149. LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
  150. LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
  151. LRU_UNEVICTABLE,
  152. NR_LRU_LISTS
  153. };
  154. #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
  155. #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
  156. static inline int is_file_lru(enum lru_list lru)
  157. {
  158. return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
  159. }
  160. static inline int is_active_lru(enum lru_list lru)
  161. {
  162. return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
  163. }
  164. static inline int is_unevictable_lru(enum lru_list lru)
  165. {
  166. return (lru == LRU_UNEVICTABLE);
  167. }
  168. struct zone_reclaim_stat {
  169. /*
  170. * The pageout code in vmscan.c keeps track of how many of the
  171. * mem/swap backed and file backed pages are referenced.
  172. * The higher the rotated/scanned ratio, the more valuable
  173. * that cache is.
  174. *
  175. * The anon LRU stats live in [0], file LRU stats in [1]
  176. */
  177. unsigned long recent_rotated[2];
  178. unsigned long recent_scanned[2];
  179. };
  180. struct lruvec {
  181. struct list_head lists[NR_LRU_LISTS];
  182. struct zone_reclaim_stat reclaim_stat;
  183. #ifdef CONFIG_MEMCG
  184. struct zone *zone;
  185. #endif
  186. };
  187. /* Mask used at gathering information at once (see memcontrol.c) */
  188. #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
  189. #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
  190. #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
  191. /* Isolate clean file */
  192. #define ISOLATE_CLEAN ((__force isolate_mode_t)0x1)
  193. /* Isolate unmapped file */
  194. #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
  195. /* Isolate for asynchronous migration */
  196. #define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
  197. /* Isolate unevictable pages */
  198. #define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
  199. /* LRU Isolation modes. */
  200. typedef unsigned __bitwise__ isolate_mode_t;
  201. enum zone_watermarks {
  202. WMARK_MIN,
  203. WMARK_LOW,
  204. WMARK_HIGH,
  205. NR_WMARK
  206. };
  207. #define min_wmark_pages(z) (z->watermark[WMARK_MIN])
  208. #define low_wmark_pages(z) (z->watermark[WMARK_LOW])
  209. #define high_wmark_pages(z) (z->watermark[WMARK_HIGH])
  210. struct per_cpu_pages {
  211. int count; /* number of pages in the list */
  212. int high; /* high watermark, emptying needed */
  213. int batch; /* chunk size for buddy add/remove */
  214. /* Lists of pages, one per migrate type stored on the pcp-lists */
  215. struct list_head lists[MIGRATE_PCPTYPES];
  216. };
  217. struct per_cpu_pageset {
  218. struct per_cpu_pages pcp;
  219. #ifdef CONFIG_NUMA
  220. s8 expire;
  221. #endif
  222. #ifdef CONFIG_SMP
  223. s8 stat_threshold;
  224. s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
  225. #endif
  226. };
  227. #endif /* !__GENERATING_BOUNDS.H */
  228. enum zone_type {
  229. #ifdef CONFIG_ZONE_DMA
  230. /*
  231. * ZONE_DMA is used when there are devices that are not able
  232. * to do DMA to all of addressable memory (ZONE_NORMAL). Then we
  233. * carve out the portion of memory that is needed for these devices.
  234. * The range is arch specific.
  235. *
  236. * Some examples
  237. *
  238. * Architecture Limit
  239. * ---------------------------
  240. * parisc, ia64, sparc <4G
  241. * s390 <2G
  242. * arm Various
  243. * alpha Unlimited or 0-16MB.
  244. *
  245. * i386, x86_64 and multiple other arches
  246. * <16M.
  247. */
  248. ZONE_DMA,
  249. #endif
  250. #ifdef CONFIG_ZONE_DMA32
  251. /*
  252. * x86_64 needs two ZONE_DMAs because it supports devices that are
  253. * only able to do DMA to the lower 16M but also 32 bit devices that
  254. * can only do DMA areas below 4G.
  255. */
  256. ZONE_DMA32,
  257. #endif
  258. /*
  259. * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
  260. * performed on pages in ZONE_NORMAL if the DMA devices support
  261. * transfers to all addressable memory.
  262. */
  263. ZONE_NORMAL,
  264. #ifdef CONFIG_HIGHMEM
  265. /*
  266. * A memory area that is only addressable by the kernel through
  267. * mapping portions into its own address space. This is for example
  268. * used by i386 to allow the kernel to address the memory beyond
  269. * 900MB. The kernel will set up special mappings (page
  270. * table entries on i386) for each page that the kernel needs to
  271. * access.
  272. */
  273. ZONE_HIGHMEM,
  274. #endif
  275. ZONE_MOVABLE,
  276. __MAX_NR_ZONES
  277. };
  278. #ifndef __GENERATING_BOUNDS_H
  279. struct zone {
  280. /* Fields commonly accessed by the page allocator */
  281. /* zone watermarks, access with *_wmark_pages(zone) macros */
  282. unsigned long watermark[NR_WMARK];
  283. /*
  284. * When free pages are below this point, additional steps are taken
  285. * when reading the number of free pages to avoid per-cpu counter
  286. * drift allowing watermarks to be breached
  287. */
  288. unsigned long percpu_drift_mark;
  289. /*
  290. * We don't know if the memory that we're going to allocate will be freeable
  291. * or/and it will be released eventually, so to avoid totally wasting several
  292. * GB of ram we must reserve some of the lower zone memory (otherwise we risk
  293. * to run OOM on the lower zones despite there's tons of freeable ram
  294. * on the higher zones). This array is recalculated at runtime if the
  295. * sysctl_lowmem_reserve_ratio sysctl changes.
  296. */
  297. unsigned long lowmem_reserve[MAX_NR_ZONES];
  298. /*
  299. * This is a per-zone reserve of pages that should not be
  300. * considered dirtyable memory.
  301. */
  302. unsigned long dirty_balance_reserve;
  303. #ifdef CONFIG_NUMA
  304. int node;
  305. /*
  306. * zone reclaim becomes active if more unmapped pages exist.
  307. */
  308. unsigned long min_unmapped_pages;
  309. unsigned long min_slab_pages;
  310. #endif
  311. struct per_cpu_pageset __percpu *pageset;
  312. /*
  313. * free areas of different sizes
  314. */
  315. spinlock_t lock;
  316. int all_unreclaimable; /* All pages pinned */
  317. #if defined CONFIG_COMPACTION || defined CONFIG_CMA
  318. /* Set to true when the PG_migrate_skip bits should be cleared */
  319. bool compact_blockskip_flush;
  320. /* pfns where compaction scanners should start */
  321. unsigned long compact_cached_free_pfn;
  322. unsigned long compact_cached_migrate_pfn;
  323. #endif
  324. #ifdef CONFIG_MEMORY_HOTPLUG
  325. /* see spanned/present_pages for more description */
  326. seqlock_t span_seqlock;
  327. #endif
  328. struct free_area free_area[MAX_ORDER];
  329. #ifndef CONFIG_SPARSEMEM
  330. /*
  331. * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
  332. * In SPARSEMEM, this map is stored in struct mem_section
  333. */
  334. unsigned long *pageblock_flags;
  335. #endif /* CONFIG_SPARSEMEM */
  336. #ifdef CONFIG_COMPACTION
  337. /*
  338. * On compaction failure, 1<<compact_defer_shift compactions
  339. * are skipped before trying again. The number attempted since
  340. * last failure is tracked with compact_considered.
  341. */
  342. unsigned int compact_considered;
  343. unsigned int compact_defer_shift;
  344. int compact_order_failed;
  345. #endif
  346. ZONE_PADDING(_pad1_)
  347. /* Fields commonly accessed by the page reclaim scanner */
  348. spinlock_t lru_lock;
  349. struct lruvec lruvec;
  350. unsigned long pages_scanned; /* since last reclaim */
  351. unsigned long flags; /* zone flags, see below */
  352. /* Zone statistics */
  353. atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
  354. /*
  355. * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on
  356. * this zone's LRU. Maintained by the pageout code.
  357. */
  358. unsigned int inactive_ratio;
  359. ZONE_PADDING(_pad2_)
  360. /* Rarely used or read-mostly fields */
  361. /*
  362. * wait_table -- the array holding the hash table
  363. * wait_table_hash_nr_entries -- the size of the hash table array
  364. * wait_table_bits -- wait_table_size == (1 << wait_table_bits)
  365. *
  366. * The purpose of all these is to keep track of the people
  367. * waiting for a page to become available and make them
  368. * runnable again when possible. The trouble is that this
  369. * consumes a lot of space, especially when so few things
  370. * wait on pages at a given time. So instead of using
  371. * per-page waitqueues, we use a waitqueue hash table.
  372. *
  373. * The bucket discipline is to sleep on the same queue when
  374. * colliding and wake all in that wait queue when removing.
  375. * When something wakes, it must check to be sure its page is
  376. * truly available, a la thundering herd. The cost of a
  377. * collision is great, but given the expected load of the
  378. * table, they should be so rare as to be outweighed by the
  379. * benefits from the saved space.
  380. *
  381. * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the
  382. * primary users of these fields, and in mm/page_alloc.c
  383. * free_area_init_core() performs the initialization of them.
  384. */
  385. wait_queue_head_t * wait_table;
  386. unsigned long wait_table_hash_nr_entries;
  387. unsigned long wait_table_bits;
  388. /*
  389. * Discontig memory support fields.
  390. */
  391. struct pglist_data *zone_pgdat;
  392. /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
  393. unsigned long zone_start_pfn;
  394. /*
  395. * spanned_pages is the total pages spanned by the zone, including
  396. * holes, which is calculated as:
  397. * spanned_pages = zone_end_pfn - zone_start_pfn;
  398. *
  399. * present_pages is physical pages existing within the zone, which
  400. * is calculated as:
  401. * present_pages = spanned_pages - absent_pages(pags in holes);
  402. *
  403. * managed_pages is present pages managed by the buddy system, which
  404. * is calculated as (reserved_pages includes pages allocated by the
  405. * bootmem allocator):
  406. * managed_pages = present_pages - reserved_pages;
  407. *
  408. * So present_pages may be used by memory hotplug or memory power
  409. * management logic to figure out unmanaged pages by checking
  410. * (present_pages - managed_pages). And managed_pages should be used
  411. * by page allocator and vm scanner to calculate all kinds of watermarks
  412. * and thresholds.
  413. *
  414. * Locking rules:
  415. *
  416. * zone_start_pfn and spanned_pages are protected by span_seqlock.
  417. * It is a seqlock because it has to be read outside of zone->lock,
  418. * and it is done in the main allocator path. But, it is written
  419. * quite infrequently.
  420. *
  421. * The span_seq lock is declared along with zone->lock because it is
  422. * frequently read in proximity to zone->lock. It's good to
  423. * give them a chance of being in the same cacheline.
  424. *
  425. * Write access to present_pages and managed_pages at runtime should
  426. * be protected by lock_memory_hotplug()/unlock_memory_hotplug().
  427. * Any reader who can't tolerant drift of present_pages and
  428. * managed_pages should hold memory hotplug lock to get a stable value.
  429. */
  430. unsigned long spanned_pages;
  431. unsigned long present_pages;
  432. unsigned long managed_pages;
  433. /*
  434. * rarely used fields:
  435. */
  436. const char *name;
  437. } ____cacheline_internodealigned_in_smp;
  438. typedef enum {
  439. ZONE_RECLAIM_LOCKED, /* prevents concurrent reclaim */
  440. ZONE_OOM_LOCKED, /* zone is in OOM killer zonelist */
  441. ZONE_CONGESTED, /* zone has many dirty pages backed by
  442. * a congested BDI
  443. */
  444. } zone_flags_t;
  445. static inline void zone_set_flag(struct zone *zone, zone_flags_t flag)
  446. {
  447. set_bit(flag, &zone->flags);
  448. }
  449. static inline int zone_test_and_set_flag(struct zone *zone, zone_flags_t flag)
  450. {
  451. return test_and_set_bit(flag, &zone->flags);
  452. }
  453. static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag)
  454. {
  455. clear_bit(flag, &zone->flags);
  456. }
  457. static inline int zone_is_reclaim_congested(const struct zone *zone)
  458. {
  459. return test_bit(ZONE_CONGESTED, &zone->flags);
  460. }
  461. static inline int zone_is_reclaim_locked(const struct zone *zone)
  462. {
  463. return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);
  464. }
  465. static inline int zone_is_oom_locked(const struct zone *zone)
  466. {
  467. return test_bit(ZONE_OOM_LOCKED, &zone->flags);
  468. }
  469. static inline unsigned zone_end_pfn(const struct zone *zone)
  470. {
  471. return zone->zone_start_pfn + zone->spanned_pages;
  472. }
  473. static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
  474. {
  475. return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
  476. }
  477. /*
  478. * The "priority" of VM scanning is how much of the queues we will scan in one
  479. * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
  480. * queues ("queue_length >> 12") during an aging round.
  481. */
  482. #define DEF_PRIORITY 12
  483. /* Maximum number of zones on a zonelist */
  484. #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
  485. #ifdef CONFIG_NUMA
  486. /*
  487. * The NUMA zonelists are doubled because we need zonelists that restrict the
  488. * allocations to a single node for GFP_THISNODE.
  489. *
  490. * [0] : Zonelist with fallback
  491. * [1] : No fallback (GFP_THISNODE)
  492. */
  493. #define MAX_ZONELISTS 2
  494. /*
  495. * We cache key information from each zonelist for smaller cache
  496. * footprint when scanning for free pages in get_page_from_freelist().
  497. *
  498. * 1) The BITMAP fullzones tracks which zones in a zonelist have come
  499. * up short of free memory since the last time (last_fullzone_zap)
  500. * we zero'd fullzones.
  501. * 2) The array z_to_n[] maps each zone in the zonelist to its node
  502. * id, so that we can efficiently evaluate whether that node is
  503. * set in the current tasks mems_allowed.
  504. *
  505. * Both fullzones and z_to_n[] are one-to-one with the zonelist,
  506. * indexed by a zones offset in the zonelist zones[] array.
  507. *
  508. * The get_page_from_freelist() routine does two scans. During the
  509. * first scan, we skip zones whose corresponding bit in 'fullzones'
  510. * is set or whose corresponding node in current->mems_allowed (which
  511. * comes from cpusets) is not set. During the second scan, we bypass
  512. * this zonelist_cache, to ensure we look methodically at each zone.
  513. *
  514. * Once per second, we zero out (zap) fullzones, forcing us to
  515. * reconsider nodes that might have regained more free memory.
  516. * The field last_full_zap is the time we last zapped fullzones.
  517. *
  518. * This mechanism reduces the amount of time we waste repeatedly
  519. * reexaming zones for free memory when they just came up low on
  520. * memory momentarilly ago.
  521. *
  522. * The zonelist_cache struct members logically belong in struct
  523. * zonelist. However, the mempolicy zonelists constructed for
  524. * MPOL_BIND are intentionally variable length (and usually much
  525. * shorter). A general purpose mechanism for handling structs with
  526. * multiple variable length members is more mechanism than we want
  527. * here. We resort to some special case hackery instead.
  528. *
  529. * The MPOL_BIND zonelists don't need this zonelist_cache (in good
  530. * part because they are shorter), so we put the fixed length stuff
  531. * at the front of the zonelist struct, ending in a variable length
  532. * zones[], as is needed by MPOL_BIND.
  533. *
  534. * Then we put the optional zonelist cache on the end of the zonelist
  535. * struct. This optional stuff is found by a 'zlcache_ptr' pointer in
  536. * the fixed length portion at the front of the struct. This pointer
  537. * both enables us to find the zonelist cache, and in the case of
  538. * MPOL_BIND zonelists, (which will just set the zlcache_ptr to NULL)
  539. * to know that the zonelist cache is not there.
  540. *
  541. * The end result is that struct zonelists come in two flavors:
  542. * 1) The full, fixed length version, shown below, and
  543. * 2) The custom zonelists for MPOL_BIND.
  544. * The custom MPOL_BIND zonelists have a NULL zlcache_ptr and no zlcache.
  545. *
  546. * Even though there may be multiple CPU cores on a node modifying
  547. * fullzones or last_full_zap in the same zonelist_cache at the same
  548. * time, we don't lock it. This is just hint data - if it is wrong now
  549. * and then, the allocator will still function, perhaps a bit slower.
  550. */
  551. struct zonelist_cache {
  552. unsigned short z_to_n[MAX_ZONES_PER_ZONELIST]; /* zone->nid */
  553. DECLARE_BITMAP(fullzones, MAX_ZONES_PER_ZONELIST); /* zone full? */
  554. unsigned long last_full_zap; /* when last zap'd (jiffies) */
  555. };
  556. #else
  557. #define MAX_ZONELISTS 1
  558. struct zonelist_cache;
  559. #endif
  560. /*
  561. * This struct contains information about a zone in a zonelist. It is stored
  562. * here to avoid dereferences into large structures and lookups of tables
  563. */
  564. struct zoneref {
  565. struct zone *zone; /* Pointer to actual zone */
  566. int zone_idx; /* zone_idx(zoneref->zone) */
  567. };
  568. /*
  569. * One allocation request operates on a zonelist. A zonelist
  570. * is a list of zones, the first one is the 'goal' of the
  571. * allocation, the other zones are fallback zones, in decreasing
  572. * priority.
  573. *
  574. * If zlcache_ptr is not NULL, then it is just the address of zlcache,
  575. * as explained above. If zlcache_ptr is NULL, there is no zlcache.
  576. * *
  577. * To speed the reading of the zonelist, the zonerefs contain the zone index
  578. * of the entry being read. Helper functions to access information given
  579. * a struct zoneref are
  580. *
  581. * zonelist_zone() - Return the struct zone * for an entry in _zonerefs
  582. * zonelist_zone_idx() - Return the index of the zone for an entry
  583. * zonelist_node_idx() - Return the index of the node for an entry
  584. */
  585. struct zonelist {
  586. struct zonelist_cache *zlcache_ptr; // NULL or &zlcache
  587. struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
  588. #ifdef CONFIG_NUMA
  589. struct zonelist_cache zlcache; // optional ...
  590. #endif
  591. };
  592. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  593. struct node_active_region {
  594. unsigned long start_pfn;
  595. unsigned long end_pfn;
  596. int nid;
  597. };
  598. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  599. #ifndef CONFIG_DISCONTIGMEM
  600. /* The array of struct pages - for discontigmem use pgdat->lmem_map */
  601. extern struct page *mem_map;
  602. #endif
  603. /*
  604. * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM
  605. * (mostly NUMA machines?) to denote a higher-level memory zone than the
  606. * zone denotes.
  607. *
  608. * On NUMA machines, each NUMA node would have a pg_data_t to describe
  609. * it's memory layout.
  610. *
  611. * Memory statistics and page replacement data structures are maintained on a
  612. * per-zone basis.
  613. */
  614. struct bootmem_data;
  615. typedef struct pglist_data {
  616. struct zone node_zones[MAX_NR_ZONES];
  617. struct zonelist node_zonelists[MAX_ZONELISTS];
  618. int nr_zones;
  619. #ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */
  620. struct page *node_mem_map;
  621. #ifdef CONFIG_MEMCG
  622. struct page_cgroup *node_page_cgroup;
  623. #endif
  624. #endif
  625. #ifndef CONFIG_NO_BOOTMEM
  626. struct bootmem_data *bdata;
  627. #endif
  628. #ifdef CONFIG_MEMORY_HOTPLUG
  629. /*
  630. * Must be held any time you expect node_start_pfn, node_present_pages
  631. * or node_spanned_pages stay constant. Holding this will also
  632. * guarantee that any pfn_valid() stays that way.
  633. *
  634. * Nests above zone->lock and zone->size_seqlock.
  635. */
  636. spinlock_t node_size_lock;
  637. #endif
  638. unsigned long node_start_pfn;
  639. unsigned long node_present_pages; /* total number of physical pages */
  640. unsigned long node_spanned_pages; /* total size of physical page
  641. range, including holes */
  642. int node_id;
  643. nodemask_t reclaim_nodes; /* Nodes allowed to reclaim from */
  644. wait_queue_head_t kswapd_wait;
  645. wait_queue_head_t pfmemalloc_wait;
  646. struct task_struct *kswapd; /* Protected by lock_memory_hotplug() */
  647. int kswapd_max_order;
  648. enum zone_type classzone_idx;
  649. #ifdef CONFIG_NUMA_BALANCING
  650. /*
  651. * Lock serializing the per destination node AutoNUMA memory
  652. * migration rate limiting data.
  653. */
  654. spinlock_t numabalancing_migrate_lock;
  655. /* Rate limiting time interval */
  656. unsigned long numabalancing_migrate_next_window;
  657. /* Number of pages migrated during the rate limiting time interval */
  658. unsigned long numabalancing_migrate_nr_pages;
  659. #endif
  660. } pg_data_t;
  661. #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
  662. #define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
  663. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  664. #define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr))
  665. #else
  666. #define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr))
  667. #endif
  668. #define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
  669. #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
  670. #define node_end_pfn(nid) ({\
  671. pg_data_t *__pgdat = NODE_DATA(nid);\
  672. __pgdat->node_start_pfn + __pgdat->node_spanned_pages;\
  673. })
  674. #include <linux/memory_hotplug.h>
  675. extern struct mutex zonelists_mutex;
  676. void build_all_zonelists(pg_data_t *pgdat, struct zone *zone);
  677. void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx);
  678. bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  679. int classzone_idx, int alloc_flags);
  680. bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
  681. int classzone_idx, int alloc_flags);
  682. enum memmap_context {
  683. MEMMAP_EARLY,
  684. MEMMAP_HOTPLUG,
  685. };
  686. extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
  687. unsigned long size,
  688. enum memmap_context context);
  689. extern void lruvec_init(struct lruvec *lruvec);
  690. static inline struct zone *lruvec_zone(struct lruvec *lruvec)
  691. {
  692. #ifdef CONFIG_MEMCG
  693. return lruvec->zone;
  694. #else
  695. return container_of(lruvec, struct zone, lruvec);
  696. #endif
  697. }
  698. #ifdef CONFIG_HAVE_MEMORY_PRESENT
  699. void memory_present(int nid, unsigned long start, unsigned long end);
  700. #else
  701. static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
  702. #endif
  703. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  704. int local_memory_node(int node_id);
  705. #else
  706. static inline int local_memory_node(int node_id) { return node_id; };
  707. #endif
  708. #ifdef CONFIG_NEED_NODE_MEMMAP_SIZE
  709. unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
  710. #endif
  711. /*
  712. * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
  713. */
  714. #define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
  715. static inline int populated_zone(struct zone *zone)
  716. {
  717. return (!!zone->present_pages);
  718. }
  719. extern int movable_zone;
  720. static inline int zone_movable_is_highmem(void)
  721. {
  722. #if defined(CONFIG_HIGHMEM) && defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
  723. return movable_zone == ZONE_HIGHMEM;
  724. #else
  725. return 0;
  726. #endif
  727. }
  728. static inline int is_highmem_idx(enum zone_type idx)
  729. {
  730. #ifdef CONFIG_HIGHMEM
  731. return (idx == ZONE_HIGHMEM ||
  732. (idx == ZONE_MOVABLE && zone_movable_is_highmem()));
  733. #else
  734. return 0;
  735. #endif
  736. }
  737. static inline int is_normal_idx(enum zone_type idx)
  738. {
  739. return (idx == ZONE_NORMAL);
  740. }
  741. /**
  742. * is_highmem - helper function to quickly check if a struct zone is a
  743. * highmem zone or not. This is an attempt to keep references
  744. * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
  745. * @zone - pointer to struct zone variable
  746. */
  747. static inline int is_highmem(struct zone *zone)
  748. {
  749. #ifdef CONFIG_HIGHMEM
  750. int zone_off = (char *)zone - (char *)zone->zone_pgdat->node_zones;
  751. return zone_off == ZONE_HIGHMEM * sizeof(*zone) ||
  752. (zone_off == ZONE_MOVABLE * sizeof(*zone) &&
  753. zone_movable_is_highmem());
  754. #else
  755. return 0;
  756. #endif
  757. }
  758. static inline int is_normal(struct zone *zone)
  759. {
  760. return zone == zone->zone_pgdat->node_zones + ZONE_NORMAL;
  761. }
  762. static inline int is_dma32(struct zone *zone)
  763. {
  764. #ifdef CONFIG_ZONE_DMA32
  765. return zone == zone->zone_pgdat->node_zones + ZONE_DMA32;
  766. #else
  767. return 0;
  768. #endif
  769. }
  770. static inline int is_dma(struct zone *zone)
  771. {
  772. #ifdef CONFIG_ZONE_DMA
  773. return zone == zone->zone_pgdat->node_zones + ZONE_DMA;
  774. #else
  775. return 0;
  776. #endif
  777. }
  778. /* These two functions are used to setup the per zone pages min values */
  779. struct ctl_table;
  780. int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
  781. void __user *, size_t *, loff_t *);
  782. extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
  783. int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
  784. void __user *, size_t *, loff_t *);
  785. int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int,
  786. void __user *, size_t *, loff_t *);
  787. int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
  788. void __user *, size_t *, loff_t *);
  789. int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
  790. void __user *, size_t *, loff_t *);
  791. extern int numa_zonelist_order_handler(struct ctl_table *, int,
  792. void __user *, size_t *, loff_t *);
  793. extern char numa_zonelist_order[];
  794. #define NUMA_ZONELIST_ORDER_LEN 16 /* string buffer size */
  795. #ifndef CONFIG_NEED_MULTIPLE_NODES
  796. extern struct pglist_data contig_page_data;
  797. #define NODE_DATA(nid) (&contig_page_data)
  798. #define NODE_MEM_MAP(nid) mem_map
  799. #else /* CONFIG_NEED_MULTIPLE_NODES */
  800. #include <asm/mmzone.h>
  801. #endif /* !CONFIG_NEED_MULTIPLE_NODES */
  802. extern struct pglist_data *first_online_pgdat(void);
  803. extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
  804. extern struct zone *next_zone(struct zone *zone);
  805. /**
  806. * for_each_online_pgdat - helper macro to iterate over all online nodes
  807. * @pgdat - pointer to a pg_data_t variable
  808. */
  809. #define for_each_online_pgdat(pgdat) \
  810. for (pgdat = first_online_pgdat(); \
  811. pgdat; \
  812. pgdat = next_online_pgdat(pgdat))
  813. /**
  814. * for_each_zone - helper macro to iterate over all memory zones
  815. * @zone - pointer to struct zone variable
  816. *
  817. * The user only needs to declare the zone variable, for_each_zone
  818. * fills it in.
  819. */
  820. #define for_each_zone(zone) \
  821. for (zone = (first_online_pgdat())->node_zones; \
  822. zone; \
  823. zone = next_zone(zone))
  824. #define for_each_populated_zone(zone) \
  825. for (zone = (first_online_pgdat())->node_zones; \
  826. zone; \
  827. zone = next_zone(zone)) \
  828. if (!populated_zone(zone)) \
  829. ; /* do nothing */ \
  830. else
  831. static inline struct zone *zonelist_zone(struct zoneref *zoneref)
  832. {
  833. return zoneref->zone;
  834. }
  835. static inline int zonelist_zone_idx(struct zoneref *zoneref)
  836. {
  837. return zoneref->zone_idx;
  838. }
  839. static inline int zonelist_node_idx(struct zoneref *zoneref)
  840. {
  841. #ifdef CONFIG_NUMA
  842. /* zone_to_nid not available in this context */
  843. return zoneref->zone->node;
  844. #else
  845. return 0;
  846. #endif /* CONFIG_NUMA */
  847. }
  848. /**
  849. * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
  850. * @z - The cursor used as a starting point for the search
  851. * @highest_zoneidx - The zone index of the highest zone to return
  852. * @nodes - An optional nodemask to filter the zonelist with
  853. * @zone - The first suitable zone found is returned via this parameter
  854. *
  855. * This function returns the next zone at or below a given zone index that is
  856. * within the allowed nodemask using a cursor as the starting point for the
  857. * search. The zoneref returned is a cursor that represents the current zone
  858. * being examined. It should be advanced by one before calling
  859. * next_zones_zonelist again.
  860. */
  861. struct zoneref *next_zones_zonelist(struct zoneref *z,
  862. enum zone_type highest_zoneidx,
  863. nodemask_t *nodes,
  864. struct zone **zone);
  865. /**
  866. * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
  867. * @zonelist - The zonelist to search for a suitable zone
  868. * @highest_zoneidx - The zone index of the highest zone to return
  869. * @nodes - An optional nodemask to filter the zonelist with
  870. * @zone - The first suitable zone found is returned via this parameter
  871. *
  872. * This function returns the first zone at or below a given zone index that is
  873. * within the allowed nodemask. The zoneref returned is a cursor that can be
  874. * used to iterate the zonelist with next_zones_zonelist by advancing it by
  875. * one before calling.
  876. */
  877. static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
  878. enum zone_type highest_zoneidx,
  879. nodemask_t *nodes,
  880. struct zone **zone)
  881. {
  882. return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes,
  883. zone);
  884. }
  885. /**
  886. * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
  887. * @zone - The current zone in the iterator
  888. * @z - The current pointer within zonelist->zones being iterated
  889. * @zlist - The zonelist being iterated
  890. * @highidx - The zone index of the highest zone to return
  891. * @nodemask - Nodemask allowed by the allocator
  892. *
  893. * This iterator iterates though all zones at or below a given zone index and
  894. * within a given nodemask
  895. */
  896. #define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
  897. for (z = first_zones_zonelist(zlist, highidx, nodemask, &zone); \
  898. zone; \
  899. z = next_zones_zonelist(++z, highidx, nodemask, &zone)) \
  900. /**
  901. * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
  902. * @zone - The current zone in the iterator
  903. * @z - The current pointer within zonelist->zones being iterated
  904. * @zlist - The zonelist being iterated
  905. * @highidx - The zone index of the highest zone to return
  906. *
  907. * This iterator iterates though all zones at or below a given zone index.
  908. */
  909. #define for_each_zone_zonelist(zone, z, zlist, highidx) \
  910. for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
  911. #ifdef CONFIG_SPARSEMEM
  912. #include <asm/sparsemem.h>
  913. #endif
  914. #if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
  915. !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
  916. static inline unsigned long early_pfn_to_nid(unsigned long pfn)
  917. {
  918. return 0;
  919. }
  920. #endif
  921. #ifdef CONFIG_FLATMEM
  922. #define pfn_to_nid(pfn) (0)
  923. #endif
  924. #ifdef CONFIG_SPARSEMEM
  925. /*
  926. * SECTION_SHIFT #bits space required to store a section #
  927. *
  928. * PA_SECTION_SHIFT physical address to/from section number
  929. * PFN_SECTION_SHIFT pfn to/from section number
  930. */
  931. #define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
  932. #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
  933. #define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
  934. #define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
  935. #define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
  936. #define SECTION_BLOCKFLAGS_BITS \
  937. ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
  938. #if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
  939. #error Allocator MAX_ORDER exceeds SECTION_SIZE
  940. #endif
  941. #define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
  942. #define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)
  943. #define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
  944. #define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
  945. struct page;
  946. struct page_cgroup;
  947. struct mem_section {
  948. /*
  949. * This is, logically, a pointer to an array of struct
  950. * pages. However, it is stored with some other magic.
  951. * (see sparse.c::sparse_init_one_section())
  952. *
  953. * Additionally during early boot we encode node id of
  954. * the location of the section here to guide allocation.
  955. * (see sparse.c::memory_present())
  956. *
  957. * Making it a UL at least makes someone do a cast
  958. * before using it wrong.
  959. */
  960. unsigned long section_mem_map;
  961. /* See declaration of similar field in struct zone */
  962. unsigned long *pageblock_flags;
  963. #ifdef CONFIG_MEMCG
  964. /*
  965. * If !SPARSEMEM, pgdat doesn't have page_cgroup pointer. We use
  966. * section. (see memcontrol.h/page_cgroup.h about this.)
  967. */
  968. struct page_cgroup *page_cgroup;
  969. unsigned long pad;
  970. #endif
  971. };
  972. #ifdef CONFIG_SPARSEMEM_EXTREME
  973. #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
  974. #else
  975. #define SECTIONS_PER_ROOT 1
  976. #endif
  977. #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
  978. #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
  979. #define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
  980. #ifdef CONFIG_SPARSEMEM_EXTREME
  981. extern struct mem_section *mem_section[NR_SECTION_ROOTS];
  982. #else
  983. extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
  984. #endif
  985. static inline struct mem_section *__nr_to_section(unsigned long nr)
  986. {
  987. if (!mem_section[SECTION_NR_TO_ROOT(nr)])
  988. return NULL;
  989. return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
  990. }
  991. extern int __section_nr(struct mem_section* ms);
  992. extern unsigned long usemap_size(void);
  993. /*
  994. * We use the lower bits of the mem_map pointer to store
  995. * a little bit of information. There should be at least
  996. * 3 bits here due to 32-bit alignment.
  997. */
  998. #define SECTION_MARKED_PRESENT (1UL<<0)
  999. #define SECTION_HAS_MEM_MAP (1UL<<1)
  1000. #define SECTION_MAP_LAST_BIT (1UL<<2)
  1001. #define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1))
  1002. #define SECTION_NID_SHIFT 2
  1003. static inline struct page *__section_mem_map_addr(struct mem_section *section)
  1004. {
  1005. unsigned long map = section->section_mem_map;
  1006. map &= SECTION_MAP_MASK;
  1007. return (struct page *)map;
  1008. }
  1009. static inline int present_section(struct mem_section *section)
  1010. {
  1011. return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
  1012. }
  1013. static inline int present_section_nr(unsigned long nr)
  1014. {
  1015. return present_section(__nr_to_section(nr));
  1016. }
  1017. static inline int valid_section(struct mem_section *section)
  1018. {
  1019. return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
  1020. }
  1021. static inline int valid_section_nr(unsigned long nr)
  1022. {
  1023. return valid_section(__nr_to_section(nr));
  1024. }
  1025. static inline struct mem_section *__pfn_to_section(unsigned long pfn)
  1026. {
  1027. return __nr_to_section(pfn_to_section_nr(pfn));
  1028. }
  1029. #ifndef CONFIG_HAVE_ARCH_PFN_VALID
  1030. static inline int pfn_valid(unsigned long pfn)
  1031. {
  1032. if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
  1033. return 0;
  1034. return valid_section(__nr_to_section(pfn_to_section_nr(pfn)));
  1035. }
  1036. #endif
  1037. static inline int pfn_present(unsigned long pfn)
  1038. {
  1039. if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
  1040. return 0;
  1041. return present_section(__nr_to_section(pfn_to_section_nr(pfn)));
  1042. }
  1043. /*
  1044. * These are _only_ used during initialisation, therefore they
  1045. * can use __initdata ... They could have names to indicate
  1046. * this restriction.
  1047. */
  1048. #ifdef CONFIG_NUMA
  1049. #define pfn_to_nid(pfn) \
  1050. ({ \
  1051. unsigned long __pfn_to_nid_pfn = (pfn); \
  1052. page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
  1053. })
  1054. #else
  1055. #define pfn_to_nid(pfn) (0)
  1056. #endif
  1057. #define early_pfn_valid(pfn) pfn_valid(pfn)
  1058. void sparse_init(void);
  1059. #else
  1060. #define sparse_init() do {} while (0)
  1061. #define sparse_index_init(_sec, _nid) do {} while (0)
  1062. #endif /* CONFIG_SPARSEMEM */
  1063. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  1064. bool early_pfn_in_nid(unsigned long pfn, int nid);
  1065. #else
  1066. #define early_pfn_in_nid(pfn, nid) (1)
  1067. #endif
  1068. #ifndef early_pfn_valid
  1069. #define early_pfn_valid(pfn) (1)
  1070. #endif
  1071. void memory_present(int nid, unsigned long start, unsigned long end);
  1072. unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
  1073. /*
  1074. * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we
  1075. * need to check pfn validility within that MAX_ORDER_NR_PAGES block.
  1076. * pfn_valid_within() should be used in this case; we optimise this away
  1077. * when we have no holes within a MAX_ORDER_NR_PAGES block.
  1078. */
  1079. #ifdef CONFIG_HOLES_IN_ZONE
  1080. #define pfn_valid_within(pfn) pfn_valid(pfn)
  1081. #else
  1082. #define pfn_valid_within(pfn) (1)
  1083. #endif
  1084. #ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL
  1085. /*
  1086. * pfn_valid() is meant to be able to tell if a given PFN has valid memmap
  1087. * associated with it or not. In FLATMEM, it is expected that holes always
  1088. * have valid memmap as long as there is valid PFNs either side of the hole.
  1089. * In SPARSEMEM, it is assumed that a valid section has a memmap for the
  1090. * entire section.
  1091. *
  1092. * However, an ARM, and maybe other embedded architectures in the future
  1093. * free memmap backing holes to save memory on the assumption the memmap is
  1094. * never used. The page_zone linkages are then broken even though pfn_valid()
  1095. * returns true. A walker of the full memmap must then do this additional
  1096. * check to ensure the memmap they are looking at is sane by making sure
  1097. * the zone and PFN linkages are still valid. This is expensive, but walkers
  1098. * of the full memmap are extremely rare.
  1099. */
  1100. int memmap_valid_within(unsigned long pfn,
  1101. struct page *page, struct zone *zone);
  1102. #else
  1103. static inline int memmap_valid_within(unsigned long pfn,
  1104. struct page *page, struct zone *zone)
  1105. {
  1106. return 1;
  1107. }
  1108. #endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */
  1109. #endif /* !__GENERATING_BOUNDS.H */
  1110. #endif /* !__ASSEMBLY__ */
  1111. #endif /* _LINUX_MMZONE_H */