sparse.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830
  1. /*
  2. * sparse memory mappings.
  3. */
  4. #include <linux/mm.h>
  5. #include <linux/slab.h>
  6. #include <linux/mmzone.h>
  7. #include <linux/bootmem.h>
  8. #include <linux/highmem.h>
  9. #include <linux/export.h>
  10. #include <linux/spinlock.h>
  11. #include <linux/vmalloc.h>
  12. #include "internal.h"
  13. #include <asm/dma.h>
  14. #include <asm/pgalloc.h>
  15. #include <asm/pgtable.h>
  16. /*
  17. * Permanent SPARSEMEM data:
  18. *
  19. * 1) mem_section - memory sections, mem_map's for valid memory
  20. */
  21. #ifdef CONFIG_SPARSEMEM_EXTREME
  22. struct mem_section *mem_section[NR_SECTION_ROOTS]
  23. ____cacheline_internodealigned_in_smp;
  24. #else
  25. struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
  26. ____cacheline_internodealigned_in_smp;
  27. #endif
  28. EXPORT_SYMBOL(mem_section);
  29. #ifdef NODE_NOT_IN_PAGE_FLAGS
  30. /*
  31. * If we did not store the node number in the page then we have to
  32. * do a lookup in the section_to_node_table in order to find which
  33. * node the page belongs to.
  34. */
  35. #if MAX_NUMNODES <= 256
  36. static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
  37. #else
  38. static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
  39. #endif
  40. int page_to_nid(const struct page *page)
  41. {
  42. return section_to_node_table[page_to_section(page)];
  43. }
  44. EXPORT_SYMBOL(page_to_nid);
  45. static void set_section_nid(unsigned long section_nr, int nid)
  46. {
  47. section_to_node_table[section_nr] = nid;
  48. }
  49. #else /* !NODE_NOT_IN_PAGE_FLAGS */
  50. static inline void set_section_nid(unsigned long section_nr, int nid)
  51. {
  52. }
  53. #endif
  54. #ifdef CONFIG_SPARSEMEM_EXTREME
  55. static struct mem_section noinline __init_refok *sparse_index_alloc(int nid)
  56. {
  57. struct mem_section *section = NULL;
  58. unsigned long array_size = SECTIONS_PER_ROOT *
  59. sizeof(struct mem_section);
  60. if (slab_is_available()) {
  61. if (node_state(nid, N_HIGH_MEMORY))
  62. section = kzalloc_node(array_size, GFP_KERNEL, nid);
  63. else
  64. section = kzalloc(array_size, GFP_KERNEL);
  65. } else {
  66. section = alloc_bootmem_node(NODE_DATA(nid), array_size);
  67. }
  68. return section;
  69. }
  70. static int __meminit sparse_index_init(unsigned long section_nr, int nid)
  71. {
  72. unsigned long root = SECTION_NR_TO_ROOT(section_nr);
  73. struct mem_section *section;
  74. if (mem_section[root])
  75. return -EEXIST;
  76. section = sparse_index_alloc(nid);
  77. if (!section)
  78. return -ENOMEM;
  79. mem_section[root] = section;
  80. return 0;
  81. }
  82. #else /* !SPARSEMEM_EXTREME */
  83. static inline int sparse_index_init(unsigned long section_nr, int nid)
  84. {
  85. return 0;
  86. }
  87. #endif
  88. /*
  89. * Although written for the SPARSEMEM_EXTREME case, this happens
  90. * to also work for the flat array case because
  91. * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
  92. */
  93. int __section_nr(struct mem_section* ms)
  94. {
  95. unsigned long root_nr;
  96. struct mem_section* root;
  97. for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
  98. root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
  99. if (!root)
  100. continue;
  101. if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
  102. break;
  103. }
  104. VM_BUG_ON(root_nr == NR_SECTION_ROOTS);
  105. return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
  106. }
  107. /*
  108. * During early boot, before section_mem_map is used for an actual
  109. * mem_map, we use section_mem_map to store the section's NUMA
  110. * node. This keeps us from having to use another data structure. The
  111. * node information is cleared just before we store the real mem_map.
  112. */
  113. static inline unsigned long sparse_encode_early_nid(int nid)
  114. {
  115. return (nid << SECTION_NID_SHIFT);
  116. }
  117. static inline int sparse_early_nid(struct mem_section *section)
  118. {
  119. return (section->section_mem_map >> SECTION_NID_SHIFT);
  120. }
  121. /* Validate the physical addressing limitations of the model */
  122. void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
  123. unsigned long *end_pfn)
  124. {
  125. unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);
  126. /*
  127. * Sanity checks - do not allow an architecture to pass
  128. * in larger pfns than the maximum scope of sparsemem:
  129. */
  130. if (*start_pfn > max_sparsemem_pfn) {
  131. mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
  132. "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
  133. *start_pfn, *end_pfn, max_sparsemem_pfn);
  134. WARN_ON_ONCE(1);
  135. *start_pfn = max_sparsemem_pfn;
  136. *end_pfn = max_sparsemem_pfn;
  137. } else if (*end_pfn > max_sparsemem_pfn) {
  138. mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
  139. "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
  140. *start_pfn, *end_pfn, max_sparsemem_pfn);
  141. WARN_ON_ONCE(1);
  142. *end_pfn = max_sparsemem_pfn;
  143. }
  144. }
  145. /* Record a memory area against a node. */
  146. void __init memory_present(int nid, unsigned long start, unsigned long end)
  147. {
  148. unsigned long pfn;
  149. start &= PAGE_SECTION_MASK;
  150. mminit_validate_memmodel_limits(&start, &end);
  151. for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
  152. unsigned long section = pfn_to_section_nr(pfn);
  153. struct mem_section *ms;
  154. sparse_index_init(section, nid);
  155. set_section_nid(section, nid);
  156. ms = __nr_to_section(section);
  157. if (!ms->section_mem_map)
  158. ms->section_mem_map = sparse_encode_early_nid(nid) |
  159. SECTION_MARKED_PRESENT;
  160. }
  161. }
  162. /*
  163. * Only used by the i386 NUMA architecures, but relatively
  164. * generic code.
  165. */
  166. unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
  167. unsigned long end_pfn)
  168. {
  169. unsigned long pfn;
  170. unsigned long nr_pages = 0;
  171. mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
  172. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  173. if (nid != early_pfn_to_nid(pfn))
  174. continue;
  175. if (pfn_present(pfn))
  176. nr_pages += PAGES_PER_SECTION;
  177. }
  178. return nr_pages * sizeof(struct page);
  179. }
  180. /*
  181. * Subtle, we encode the real pfn into the mem_map such that
  182. * the identity pfn - section_mem_map will return the actual
  183. * physical page frame number.
  184. */
  185. static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
  186. {
  187. return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
  188. }
  189. /*
  190. * Decode mem_map from the coded memmap
  191. */
  192. struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
  193. {
  194. /* mask off the extra low bits of information */
  195. coded_mem_map &= SECTION_MAP_MASK;
  196. return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
  197. }
  198. static int __meminit sparse_init_one_section(struct mem_section *ms,
  199. unsigned long pnum, struct page *mem_map,
  200. unsigned long *pageblock_bitmap)
  201. {
  202. if (!present_section(ms))
  203. return -EINVAL;
  204. ms->section_mem_map &= ~SECTION_MAP_MASK;
  205. ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
  206. SECTION_HAS_MEM_MAP;
  207. ms->pageblock_flags = pageblock_bitmap;
  208. return 1;
  209. }
  210. unsigned long usemap_size(void)
  211. {
  212. unsigned long size_bytes;
  213. size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8;
  214. size_bytes = roundup(size_bytes, sizeof(unsigned long));
  215. return size_bytes;
  216. }
  217. #ifdef CONFIG_MEMORY_HOTPLUG
  218. static unsigned long *__kmalloc_section_usemap(void)
  219. {
  220. return kmalloc(usemap_size(), GFP_KERNEL);
  221. }
  222. #endif /* CONFIG_MEMORY_HOTPLUG */
  223. #ifdef CONFIG_MEMORY_HOTREMOVE
  224. static unsigned long * __init
  225. sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
  226. unsigned long size)
  227. {
  228. unsigned long goal, limit;
  229. unsigned long *p;
  230. int nid;
  231. /*
  232. * A page may contain usemaps for other sections preventing the
  233. * page being freed and making a section unremovable while
  234. * other sections referencing the usemap retmain active. Similarly,
  235. * a pgdat can prevent a section being removed. If section A
  236. * contains a pgdat and section B contains the usemap, both
  237. * sections become inter-dependent. This allocates usemaps
  238. * from the same section as the pgdat where possible to avoid
  239. * this problem.
  240. */
  241. goal = __pa(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);
  242. limit = goal + (1UL << PA_SECTION_SHIFT);
  243. nid = early_pfn_to_nid(goal >> PAGE_SHIFT);
  244. again:
  245. p = ___alloc_bootmem_node_nopanic(NODE_DATA(nid), size,
  246. SMP_CACHE_BYTES, goal, limit);
  247. if (!p && limit) {
  248. limit = 0;
  249. goto again;
  250. }
  251. return p;
  252. }
  253. static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
  254. {
  255. unsigned long usemap_snr, pgdat_snr;
  256. static unsigned long old_usemap_snr = NR_MEM_SECTIONS;
  257. static unsigned long old_pgdat_snr = NR_MEM_SECTIONS;
  258. struct pglist_data *pgdat = NODE_DATA(nid);
  259. int usemap_nid;
  260. usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);
  261. pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
  262. if (usemap_snr == pgdat_snr)
  263. return;
  264. if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
  265. /* skip redundant message */
  266. return;
  267. old_usemap_snr = usemap_snr;
  268. old_pgdat_snr = pgdat_snr;
  269. usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
  270. if (usemap_nid != nid) {
  271. printk(KERN_INFO
  272. "node %d must be removed before remove section %ld\n",
  273. nid, usemap_snr);
  274. return;
  275. }
  276. /*
  277. * There is a circular dependency.
  278. * Some platforms allow un-removable section because they will just
  279. * gather other removable sections for dynamic partitioning.
  280. * Just notify un-removable section's number here.
  281. */
  282. printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr,
  283. pgdat_snr, nid);
  284. printk(KERN_CONT
  285. " have a circular dependency on usemap and pgdat allocations\n");
  286. }
  287. #else
  288. static unsigned long * __init
  289. sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
  290. unsigned long size)
  291. {
  292. return alloc_bootmem_node_nopanic(pgdat, size);
  293. }
  294. static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
  295. {
  296. }
  297. #endif /* CONFIG_MEMORY_HOTREMOVE */
  298. static void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map,
  299. unsigned long pnum_begin,
  300. unsigned long pnum_end,
  301. unsigned long usemap_count, int nodeid)
  302. {
  303. void *usemap;
  304. unsigned long pnum;
  305. int size = usemap_size();
  306. usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
  307. size * usemap_count);
  308. if (!usemap) {
  309. printk(KERN_WARNING "%s: allocation failed\n", __func__);
  310. return;
  311. }
  312. for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
  313. if (!present_section_nr(pnum))
  314. continue;
  315. usemap_map[pnum] = usemap;
  316. usemap += size;
  317. check_usemap_section_nr(nodeid, usemap_map[pnum]);
  318. }
  319. }
  320. #ifndef CONFIG_SPARSEMEM_VMEMMAP
  321. struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
  322. {
  323. struct page *map;
  324. unsigned long size;
  325. map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
  326. if (map)
  327. return map;
  328. size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
  329. map = __alloc_bootmem_node_high(NODE_DATA(nid), size,
  330. PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
  331. return map;
  332. }
  333. void __init sparse_mem_maps_populate_node(struct page **map_map,
  334. unsigned long pnum_begin,
  335. unsigned long pnum_end,
  336. unsigned long map_count, int nodeid)
  337. {
  338. void *map;
  339. unsigned long pnum;
  340. unsigned long size = sizeof(struct page) * PAGES_PER_SECTION;
  341. map = alloc_remap(nodeid, size * map_count);
  342. if (map) {
  343. for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
  344. if (!present_section_nr(pnum))
  345. continue;
  346. map_map[pnum] = map;
  347. map += size;
  348. }
  349. return;
  350. }
  351. size = PAGE_ALIGN(size);
  352. map = __alloc_bootmem_node_high(NODE_DATA(nodeid), size * map_count,
  353. PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
  354. if (map) {
  355. for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
  356. if (!present_section_nr(pnum))
  357. continue;
  358. map_map[pnum] = map;
  359. map += size;
  360. }
  361. return;
  362. }
  363. /* fallback */
  364. for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
  365. struct mem_section *ms;
  366. if (!present_section_nr(pnum))
  367. continue;
  368. map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
  369. if (map_map[pnum])
  370. continue;
  371. ms = __nr_to_section(pnum);
  372. printk(KERN_ERR "%s: sparsemem memory map backing failed "
  373. "some memory will not be available.\n", __func__);
  374. ms->section_mem_map = 0;
  375. }
  376. }
  377. #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
  378. #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
  379. static void __init sparse_early_mem_maps_alloc_node(struct page **map_map,
  380. unsigned long pnum_begin,
  381. unsigned long pnum_end,
  382. unsigned long map_count, int nodeid)
  383. {
  384. sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end,
  385. map_count, nodeid);
  386. }
  387. #else
  388. static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
  389. {
  390. struct page *map;
  391. struct mem_section *ms = __nr_to_section(pnum);
  392. int nid = sparse_early_nid(ms);
  393. map = sparse_mem_map_populate(pnum, nid);
  394. if (map)
  395. return map;
  396. printk(KERN_ERR "%s: sparsemem memory map backing failed "
  397. "some memory will not be available.\n", __func__);
  398. ms->section_mem_map = 0;
  399. return NULL;
  400. }
  401. #endif
  402. void __attribute__((weak)) __meminit vmemmap_populate_print_last(void)
  403. {
  404. }
  405. /*
  406. * Allocate the accumulated non-linear sections, allocate a mem_map
  407. * for each and record the physical to section mapping.
  408. */
  409. void __init sparse_init(void)
  410. {
  411. unsigned long pnum;
  412. struct page *map;
  413. unsigned long *usemap;
  414. unsigned long **usemap_map;
  415. int size;
  416. int nodeid_begin = 0;
  417. unsigned long pnum_begin = 0;
  418. unsigned long usemap_count;
  419. #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
  420. unsigned long map_count;
  421. int size2;
  422. struct page **map_map;
  423. #endif
  424. /* see include/linux/mmzone.h 'struct mem_section' definition */
  425. BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
  426. /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
  427. set_pageblock_order();
  428. /*
  429. * map is using big page (aka 2M in x86 64 bit)
  430. * usemap is less one page (aka 24 bytes)
  431. * so alloc 2M (with 2M align) and 24 bytes in turn will
  432. * make next 2M slip to one more 2M later.
  433. * then in big system, the memory will have a lot of holes...
  434. * here try to allocate 2M pages continuously.
  435. *
  436. * powerpc need to call sparse_init_one_section right after each
  437. * sparse_early_mem_map_alloc, so allocate usemap_map at first.
  438. */
  439. size = sizeof(unsigned long *) * NR_MEM_SECTIONS;
  440. usemap_map = alloc_bootmem(size);
  441. if (!usemap_map)
  442. panic("can not allocate usemap_map\n");
  443. for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
  444. struct mem_section *ms;
  445. if (!present_section_nr(pnum))
  446. continue;
  447. ms = __nr_to_section(pnum);
  448. nodeid_begin = sparse_early_nid(ms);
  449. pnum_begin = pnum;
  450. break;
  451. }
  452. usemap_count = 1;
  453. for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
  454. struct mem_section *ms;
  455. int nodeid;
  456. if (!present_section_nr(pnum))
  457. continue;
  458. ms = __nr_to_section(pnum);
  459. nodeid = sparse_early_nid(ms);
  460. if (nodeid == nodeid_begin) {
  461. usemap_count++;
  462. continue;
  463. }
  464. /* ok, we need to take cake of from pnum_begin to pnum - 1*/
  465. sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum,
  466. usemap_count, nodeid_begin);
  467. /* new start, update count etc*/
  468. nodeid_begin = nodeid;
  469. pnum_begin = pnum;
  470. usemap_count = 1;
  471. }
  472. /* ok, last chunk */
  473. sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS,
  474. usemap_count, nodeid_begin);
  475. #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
  476. size2 = sizeof(struct page *) * NR_MEM_SECTIONS;
  477. map_map = alloc_bootmem(size2);
  478. if (!map_map)
  479. panic("can not allocate map_map\n");
  480. for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
  481. struct mem_section *ms;
  482. if (!present_section_nr(pnum))
  483. continue;
  484. ms = __nr_to_section(pnum);
  485. nodeid_begin = sparse_early_nid(ms);
  486. pnum_begin = pnum;
  487. break;
  488. }
  489. map_count = 1;
  490. for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
  491. struct mem_section *ms;
  492. int nodeid;
  493. if (!present_section_nr(pnum))
  494. continue;
  495. ms = __nr_to_section(pnum);
  496. nodeid = sparse_early_nid(ms);
  497. if (nodeid == nodeid_begin) {
  498. map_count++;
  499. continue;
  500. }
  501. /* ok, we need to take cake of from pnum_begin to pnum - 1*/
  502. sparse_early_mem_maps_alloc_node(map_map, pnum_begin, pnum,
  503. map_count, nodeid_begin);
  504. /* new start, update count etc*/
  505. nodeid_begin = nodeid;
  506. pnum_begin = pnum;
  507. map_count = 1;
  508. }
  509. /* ok, last chunk */
  510. sparse_early_mem_maps_alloc_node(map_map, pnum_begin, NR_MEM_SECTIONS,
  511. map_count, nodeid_begin);
  512. #endif
  513. for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
  514. if (!present_section_nr(pnum))
  515. continue;
  516. usemap = usemap_map[pnum];
  517. if (!usemap)
  518. continue;
  519. #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
  520. map = map_map[pnum];
  521. #else
  522. map = sparse_early_mem_map_alloc(pnum);
  523. #endif
  524. if (!map)
  525. continue;
  526. sparse_init_one_section(__nr_to_section(pnum), pnum, map,
  527. usemap);
  528. }
  529. vmemmap_populate_print_last();
  530. #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
  531. free_bootmem(__pa(map_map), size2);
  532. #endif
  533. free_bootmem(__pa(usemap_map), size);
  534. }
  535. #ifdef CONFIG_MEMORY_HOTPLUG
  536. #ifdef CONFIG_SPARSEMEM_VMEMMAP
  537. static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
  538. unsigned long nr_pages)
  539. {
  540. /* This will make the necessary allocations eventually. */
  541. return sparse_mem_map_populate(pnum, nid);
  542. }
  543. static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
  544. {
  545. unsigned long start = (unsigned long)memmap;
  546. unsigned long end = (unsigned long)(memmap + nr_pages);
  547. vmemmap_free(start, end);
  548. }
  549. #ifdef CONFIG_MEMORY_HOTREMOVE
  550. static void free_map_bootmem(struct page *memmap, unsigned long nr_pages)
  551. {
  552. unsigned long start = (unsigned long)memmap;
  553. unsigned long end = (unsigned long)(memmap + nr_pages);
  554. vmemmap_free(start, end);
  555. }
  556. #endif /* CONFIG_MEMORY_HOTREMOVE */
  557. #else
  558. static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
  559. {
  560. struct page *page, *ret;
  561. unsigned long memmap_size = sizeof(struct page) * nr_pages;
  562. page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
  563. if (page)
  564. goto got_map_page;
  565. ret = vmalloc(memmap_size);
  566. if (ret)
  567. goto got_map_ptr;
  568. return NULL;
  569. got_map_page:
  570. ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
  571. got_map_ptr:
  572. return ret;
  573. }
  574. static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
  575. unsigned long nr_pages)
  576. {
  577. return __kmalloc_section_memmap(nr_pages);
  578. }
  579. static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
  580. {
  581. if (is_vmalloc_addr(memmap))
  582. vfree(memmap);
  583. else
  584. free_pages((unsigned long)memmap,
  585. get_order(sizeof(struct page) * nr_pages));
  586. }
  587. #ifdef CONFIG_MEMORY_HOTREMOVE
  588. static void free_map_bootmem(struct page *memmap, unsigned long nr_pages)
  589. {
  590. unsigned long maps_section_nr, removing_section_nr, i;
  591. unsigned long magic;
  592. struct page *page = virt_to_page(memmap);
  593. for (i = 0; i < nr_pages; i++, page++) {
  594. magic = (unsigned long) page->lru.next;
  595. BUG_ON(magic == NODE_INFO);
  596. maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
  597. removing_section_nr = page->private;
  598. /*
  599. * When this function is called, the removing section is
  600. * logical offlined state. This means all pages are isolated
  601. * from page allocator. If removing section's memmap is placed
  602. * on the same section, it must not be freed.
  603. * If it is freed, page allocator may allocate it which will
  604. * be removed physically soon.
  605. */
  606. if (maps_section_nr != removing_section_nr)
  607. put_page_bootmem(page);
  608. }
  609. }
  610. #endif /* CONFIG_MEMORY_HOTREMOVE */
  611. #endif /* CONFIG_SPARSEMEM_VMEMMAP */
  612. /*
  613. * returns the number of sections whose mem_maps were properly
  614. * set. If this is <=0, then that means that the passed-in
  615. * map was not consumed and must be freed.
  616. */
  617. int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
  618. int nr_pages)
  619. {
  620. unsigned long section_nr = pfn_to_section_nr(start_pfn);
  621. struct pglist_data *pgdat = zone->zone_pgdat;
  622. struct mem_section *ms;
  623. struct page *memmap;
  624. unsigned long *usemap;
  625. unsigned long flags;
  626. int ret;
  627. /*
  628. * no locking for this, because it does its own
  629. * plus, it does a kmalloc
  630. */
  631. ret = sparse_index_init(section_nr, pgdat->node_id);
  632. if (ret < 0 && ret != -EEXIST)
  633. return ret;
  634. memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages);
  635. if (!memmap)
  636. return -ENOMEM;
  637. usemap = __kmalloc_section_usemap();
  638. if (!usemap) {
  639. __kfree_section_memmap(memmap, nr_pages);
  640. return -ENOMEM;
  641. }
  642. pgdat_resize_lock(pgdat, &flags);
  643. ms = __pfn_to_section(start_pfn);
  644. if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
  645. ret = -EEXIST;
  646. goto out;
  647. }
  648. memset(memmap, 0, sizeof(struct page) * nr_pages);
  649. ms->section_mem_map |= SECTION_MARKED_PRESENT;
  650. ret = sparse_init_one_section(ms, section_nr, memmap, usemap);
  651. out:
  652. pgdat_resize_unlock(pgdat, &flags);
  653. if (ret <= 0) {
  654. kfree(usemap);
  655. __kfree_section_memmap(memmap, nr_pages);
  656. }
  657. return ret;
  658. }
  659. #ifdef CONFIG_MEMORY_HOTREMOVE
  660. #ifdef CONFIG_MEMORY_FAILURE
  661. static void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
  662. {
  663. int i;
  664. if (!memmap)
  665. return;
  666. for (i = 0; i < PAGES_PER_SECTION; i++) {
  667. if (PageHWPoison(&memmap[i])) {
  668. atomic_long_sub(1, &num_poisoned_pages);
  669. ClearPageHWPoison(&memmap[i]);
  670. }
  671. }
  672. }
  673. #else
  674. static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
  675. {
  676. }
  677. #endif
  678. static void free_section_usemap(struct page *memmap, unsigned long *usemap)
  679. {
  680. struct page *usemap_page;
  681. unsigned long nr_pages;
  682. if (!usemap)
  683. return;
  684. usemap_page = virt_to_page(usemap);
  685. /*
  686. * Check to see if allocation came from hot-plug-add
  687. */
  688. if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
  689. kfree(usemap);
  690. if (memmap)
  691. __kfree_section_memmap(memmap, PAGES_PER_SECTION);
  692. return;
  693. }
  694. /*
  695. * The usemap came from bootmem. This is packed with other usemaps
  696. * on the section which has pgdat at boot time. Just keep it as is now.
  697. */
  698. if (memmap) {
  699. nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
  700. >> PAGE_SHIFT;
  701. free_map_bootmem(memmap, nr_pages);
  702. }
  703. }
  704. void sparse_remove_one_section(struct zone *zone, struct mem_section *ms)
  705. {
  706. struct page *memmap = NULL;
  707. unsigned long *usemap = NULL, flags;
  708. struct pglist_data *pgdat = zone->zone_pgdat;
  709. pgdat_resize_lock(pgdat, &flags);
  710. if (ms->section_mem_map) {
  711. usemap = ms->pageblock_flags;
  712. memmap = sparse_decode_mem_map(ms->section_mem_map,
  713. __section_nr(ms));
  714. ms->section_mem_map = 0;
  715. ms->pageblock_flags = NULL;
  716. }
  717. pgdat_resize_unlock(pgdat, &flags);
  718. clear_hwpoisoned_pages(memmap, PAGES_PER_SECTION);
  719. free_section_usemap(memmap, usemap);
  720. }
  721. #endif /* CONFIG_MEMORY_HOTREMOVE */
  722. #endif /* CONFIG_MEMORY_HOTPLUG */