memory_hotplug.c 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914
  1. /*
  2. * linux/mm/memory_hotplug.c
  3. *
  4. * Copyright (C)
  5. */
  6. #include <linux/stddef.h>
  7. #include <linux/mm.h>
  8. #include <linux/swap.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/bootmem.h>
  12. #include <linux/compiler.h>
  13. #include <linux/export.h>
  14. #include <linux/pagevec.h>
  15. #include <linux/writeback.h>
  16. #include <linux/slab.h>
  17. #include <linux/sysctl.h>
  18. #include <linux/cpu.h>
  19. #include <linux/memory.h>
  20. #include <linux/memory_hotplug.h>
  21. #include <linux/highmem.h>
  22. #include <linux/vmalloc.h>
  23. #include <linux/ioport.h>
  24. #include <linux/delay.h>
  25. #include <linux/migrate.h>
  26. #include <linux/page-isolation.h>
  27. #include <linux/pfn.h>
  28. #include <linux/suspend.h>
  29. #include <linux/mm_inline.h>
  30. #include <linux/firmware-map.h>
  31. #include <linux/stop_machine.h>
  32. #include <linux/hugetlb.h>
  33. #include <linux/memblock.h>
  34. #include <asm/tlbflush.h>
  35. #include "internal.h"
  36. /*
  37. * online_page_callback contains pointer to current page onlining function.
  38. * Initially it is generic_online_page(). If it is required it could be
  39. * changed by calling set_online_page_callback() for callback registration
  40. * and restore_online_page_callback() for generic callback restore.
  41. */
  42. static void generic_online_page(struct page *page);
  43. static online_page_callback_t online_page_callback = generic_online_page;
  44. DEFINE_MUTEX(mem_hotplug_mutex);
  45. void lock_memory_hotplug(void)
  46. {
  47. mutex_lock(&mem_hotplug_mutex);
  48. }
  49. void unlock_memory_hotplug(void)
  50. {
  51. mutex_unlock(&mem_hotplug_mutex);
  52. }
  53. /* add this memory to iomem resource */
  54. static struct resource *register_memory_resource(u64 start, u64 size)
  55. {
  56. struct resource *res;
  57. res = kzalloc(sizeof(struct resource), GFP_KERNEL);
  58. BUG_ON(!res);
  59. res->name = "System RAM";
  60. res->start = start;
  61. res->end = start + size - 1;
  62. res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
  63. if (request_resource(&iomem_resource, res) < 0) {
  64. pr_debug("System RAM resource %pR cannot be added\n", res);
  65. kfree(res);
  66. res = NULL;
  67. }
  68. return res;
  69. }
  70. static void release_memory_resource(struct resource *res)
  71. {
  72. if (!res)
  73. return;
  74. release_resource(res);
  75. kfree(res);
  76. return;
  77. }
  78. #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
  79. void get_page_bootmem(unsigned long info, struct page *page,
  80. unsigned long type)
  81. {
  82. page->lru.next = (struct list_head *) type;
  83. SetPagePrivate(page);
  84. set_page_private(page, info);
  85. atomic_inc(&page->_count);
  86. }
  87. void put_page_bootmem(struct page *page)
  88. {
  89. unsigned long type;
  90. type = (unsigned long) page->lru.next;
  91. BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
  92. type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
  93. if (atomic_dec_return(&page->_count) == 1) {
  94. ClearPagePrivate(page);
  95. set_page_private(page, 0);
  96. INIT_LIST_HEAD(&page->lru);
  97. free_reserved_page(page);
  98. }
  99. }
  100. #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
  101. #ifndef CONFIG_SPARSEMEM_VMEMMAP
  102. static void register_page_bootmem_info_section(unsigned long start_pfn)
  103. {
  104. unsigned long *usemap, mapsize, section_nr, i;
  105. struct mem_section *ms;
  106. struct page *page, *memmap;
  107. section_nr = pfn_to_section_nr(start_pfn);
  108. ms = __nr_to_section(section_nr);
  109. /* Get section's memmap address */
  110. memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  111. /*
  112. * Get page for the memmap's phys address
  113. * XXX: need more consideration for sparse_vmemmap...
  114. */
  115. page = virt_to_page(memmap);
  116. mapsize = sizeof(struct page) * PAGES_PER_SECTION;
  117. mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
  118. /* remember memmap's page */
  119. for (i = 0; i < mapsize; i++, page++)
  120. get_page_bootmem(section_nr, page, SECTION_INFO);
  121. usemap = __nr_to_section(section_nr)->pageblock_flags;
  122. page = virt_to_page(usemap);
  123. mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
  124. for (i = 0; i < mapsize; i++, page++)
  125. get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  126. }
  127. #else /* CONFIG_SPARSEMEM_VMEMMAP */
  128. static void register_page_bootmem_info_section(unsigned long start_pfn)
  129. {
  130. unsigned long *usemap, mapsize, section_nr, i;
  131. struct mem_section *ms;
  132. struct page *page, *memmap;
  133. if (!pfn_valid(start_pfn))
  134. return;
  135. section_nr = pfn_to_section_nr(start_pfn);
  136. ms = __nr_to_section(section_nr);
  137. memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  138. register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
  139. usemap = __nr_to_section(section_nr)->pageblock_flags;
  140. page = virt_to_page(usemap);
  141. mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
  142. for (i = 0; i < mapsize; i++, page++)
  143. get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  144. }
  145. #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
  146. void register_page_bootmem_info_node(struct pglist_data *pgdat)
  147. {
  148. unsigned long i, pfn, end_pfn, nr_pages;
  149. int node = pgdat->node_id;
  150. struct page *page;
  151. struct zone *zone;
  152. nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
  153. page = virt_to_page(pgdat);
  154. for (i = 0; i < nr_pages; i++, page++)
  155. get_page_bootmem(node, page, NODE_INFO);
  156. zone = &pgdat->node_zones[0];
  157. for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
  158. if (zone_is_initialized(zone)) {
  159. nr_pages = zone->wait_table_hash_nr_entries
  160. * sizeof(wait_queue_head_t);
  161. nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
  162. page = virt_to_page(zone->wait_table);
  163. for (i = 0; i < nr_pages; i++, page++)
  164. get_page_bootmem(node, page, NODE_INFO);
  165. }
  166. }
  167. pfn = pgdat->node_start_pfn;
  168. end_pfn = pgdat_end_pfn(pgdat);
  169. /* register section info */
  170. for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  171. /*
  172. * Some platforms can assign the same pfn to multiple nodes - on
  173. * node0 as well as nodeN. To avoid registering a pfn against
  174. * multiple nodes we check that this pfn does not already
  175. * reside in some other nodes.
  176. */
  177. if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node))
  178. register_page_bootmem_info_section(pfn);
  179. }
  180. }
  181. #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
  182. static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
  183. unsigned long end_pfn)
  184. {
  185. unsigned long old_zone_end_pfn;
  186. zone_span_writelock(zone);
  187. old_zone_end_pfn = zone_end_pfn(zone);
  188. if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
  189. zone->zone_start_pfn = start_pfn;
  190. zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
  191. zone->zone_start_pfn;
  192. zone_span_writeunlock(zone);
  193. }
  194. static void resize_zone(struct zone *zone, unsigned long start_pfn,
  195. unsigned long end_pfn)
  196. {
  197. zone_span_writelock(zone);
  198. if (end_pfn - start_pfn) {
  199. zone->zone_start_pfn = start_pfn;
  200. zone->spanned_pages = end_pfn - start_pfn;
  201. } else {
  202. /*
  203. * make it consist as free_area_init_core(),
  204. * if spanned_pages = 0, then keep start_pfn = 0
  205. */
  206. zone->zone_start_pfn = 0;
  207. zone->spanned_pages = 0;
  208. }
  209. zone_span_writeunlock(zone);
  210. }
  211. static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
  212. unsigned long end_pfn)
  213. {
  214. enum zone_type zid = zone_idx(zone);
  215. int nid = zone->zone_pgdat->node_id;
  216. unsigned long pfn;
  217. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  218. set_page_links(pfn_to_page(pfn), zid, nid, pfn);
  219. }
  220. /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
  221. * alloc_bootmem_node_nopanic() */
  222. static int __ref ensure_zone_is_initialized(struct zone *zone,
  223. unsigned long start_pfn, unsigned long num_pages)
  224. {
  225. if (!zone_is_initialized(zone))
  226. return init_currently_empty_zone(zone, start_pfn, num_pages,
  227. MEMMAP_HOTPLUG);
  228. return 0;
  229. }
  230. static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
  231. unsigned long start_pfn, unsigned long end_pfn)
  232. {
  233. int ret;
  234. unsigned long flags;
  235. unsigned long z1_start_pfn;
  236. ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
  237. if (ret)
  238. return ret;
  239. pgdat_resize_lock(z1->zone_pgdat, &flags);
  240. /* can't move pfns which are higher than @z2 */
  241. if (end_pfn > zone_end_pfn(z2))
  242. goto out_fail;
  243. /* the move out part must be at the left most of @z2 */
  244. if (start_pfn > z2->zone_start_pfn)
  245. goto out_fail;
  246. /* must included/overlap */
  247. if (end_pfn <= z2->zone_start_pfn)
  248. goto out_fail;
  249. /* use start_pfn for z1's start_pfn if z1 is empty */
  250. if (!zone_is_empty(z1))
  251. z1_start_pfn = z1->zone_start_pfn;
  252. else
  253. z1_start_pfn = start_pfn;
  254. resize_zone(z1, z1_start_pfn, end_pfn);
  255. resize_zone(z2, end_pfn, zone_end_pfn(z2));
  256. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  257. fix_zone_id(z1, start_pfn, end_pfn);
  258. return 0;
  259. out_fail:
  260. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  261. return -1;
  262. }
  263. static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
  264. unsigned long start_pfn, unsigned long end_pfn)
  265. {
  266. int ret;
  267. unsigned long flags;
  268. unsigned long z2_end_pfn;
  269. ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
  270. if (ret)
  271. return ret;
  272. pgdat_resize_lock(z1->zone_pgdat, &flags);
  273. /* can't move pfns which are lower than @z1 */
  274. if (z1->zone_start_pfn > start_pfn)
  275. goto out_fail;
  276. /* the move out part mast at the right most of @z1 */
  277. if (zone_end_pfn(z1) > end_pfn)
  278. goto out_fail;
  279. /* must included/overlap */
  280. if (start_pfn >= zone_end_pfn(z1))
  281. goto out_fail;
  282. /* use end_pfn for z2's end_pfn if z2 is empty */
  283. if (!zone_is_empty(z2))
  284. z2_end_pfn = zone_end_pfn(z2);
  285. else
  286. z2_end_pfn = end_pfn;
  287. resize_zone(z1, z1->zone_start_pfn, start_pfn);
  288. resize_zone(z2, start_pfn, z2_end_pfn);
  289. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  290. fix_zone_id(z2, start_pfn, end_pfn);
  291. return 0;
  292. out_fail:
  293. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  294. return -1;
  295. }
  296. static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
  297. unsigned long end_pfn)
  298. {
  299. unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat);
  300. if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
  301. pgdat->node_start_pfn = start_pfn;
  302. pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
  303. pgdat->node_start_pfn;
  304. }
  305. static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
  306. {
  307. struct pglist_data *pgdat = zone->zone_pgdat;
  308. int nr_pages = PAGES_PER_SECTION;
  309. int nid = pgdat->node_id;
  310. int zone_type;
  311. unsigned long flags;
  312. int ret;
  313. zone_type = zone - pgdat->node_zones;
  314. ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
  315. if (ret)
  316. return ret;
  317. pgdat_resize_lock(zone->zone_pgdat, &flags);
  318. grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
  319. grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
  320. phys_start_pfn + nr_pages);
  321. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  322. memmap_init_zone(nr_pages, nid, zone_type,
  323. phys_start_pfn, MEMMAP_HOTPLUG);
  324. return 0;
  325. }
  326. static int __meminit __add_section(int nid, struct zone *zone,
  327. unsigned long phys_start_pfn)
  328. {
  329. int ret;
  330. if (pfn_valid(phys_start_pfn))
  331. return -EEXIST;
  332. ret = sparse_add_one_section(zone, phys_start_pfn);
  333. if (ret < 0)
  334. return ret;
  335. ret = __add_zone(zone, phys_start_pfn);
  336. if (ret < 0)
  337. return ret;
  338. return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
  339. }
  340. /*
  341. * Reasonably generic function for adding memory. It is
  342. * expected that archs that support memory hotplug will
  343. * call this function after deciding the zone to which to
  344. * add the new pages.
  345. */
  346. int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
  347. unsigned long nr_pages)
  348. {
  349. unsigned long i;
  350. int err = 0;
  351. int start_sec, end_sec;
  352. /* during initialize mem_map, align hot-added range to section */
  353. start_sec = pfn_to_section_nr(phys_start_pfn);
  354. end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
  355. for (i = start_sec; i <= end_sec; i++) {
  356. err = __add_section(nid, zone, i << PFN_SECTION_SHIFT);
  357. /*
  358. * EEXIST is finally dealt with by ioresource collision
  359. * check. see add_memory() => register_memory_resource()
  360. * Warning will be printed if there is collision.
  361. */
  362. if (err && (err != -EEXIST))
  363. break;
  364. err = 0;
  365. }
  366. return err;
  367. }
  368. EXPORT_SYMBOL_GPL(__add_pages);
  369. #ifdef CONFIG_MEMORY_HOTREMOVE
  370. /* find the smallest valid pfn in the range [start_pfn, end_pfn) */
  371. static int find_smallest_section_pfn(int nid, struct zone *zone,
  372. unsigned long start_pfn,
  373. unsigned long end_pfn)
  374. {
  375. struct mem_section *ms;
  376. for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
  377. ms = __pfn_to_section(start_pfn);
  378. if (unlikely(!valid_section(ms)))
  379. continue;
  380. if (unlikely(pfn_to_nid(start_pfn) != nid))
  381. continue;
  382. if (zone && zone != page_zone(pfn_to_page(start_pfn)))
  383. continue;
  384. return start_pfn;
  385. }
  386. return 0;
  387. }
  388. /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
  389. static int find_biggest_section_pfn(int nid, struct zone *zone,
  390. unsigned long start_pfn,
  391. unsigned long end_pfn)
  392. {
  393. struct mem_section *ms;
  394. unsigned long pfn;
  395. /* pfn is the end pfn of a memory section. */
  396. pfn = end_pfn - 1;
  397. for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
  398. ms = __pfn_to_section(pfn);
  399. if (unlikely(!valid_section(ms)))
  400. continue;
  401. if (unlikely(pfn_to_nid(pfn) != nid))
  402. continue;
  403. if (zone && zone != page_zone(pfn_to_page(pfn)))
  404. continue;
  405. return pfn;
  406. }
  407. return 0;
  408. }
  409. static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
  410. unsigned long end_pfn)
  411. {
  412. unsigned long zone_start_pfn = zone->zone_start_pfn;
  413. unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
  414. unsigned long zone_end_pfn = z;
  415. unsigned long pfn;
  416. struct mem_section *ms;
  417. int nid = zone_to_nid(zone);
  418. zone_span_writelock(zone);
  419. if (zone_start_pfn == start_pfn) {
  420. /*
  421. * If the section is smallest section in the zone, it need
  422. * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
  423. * In this case, we find second smallest valid mem_section
  424. * for shrinking zone.
  425. */
  426. pfn = find_smallest_section_pfn(nid, zone, end_pfn,
  427. zone_end_pfn);
  428. if (pfn) {
  429. zone->zone_start_pfn = pfn;
  430. zone->spanned_pages = zone_end_pfn - pfn;
  431. }
  432. } else if (zone_end_pfn == end_pfn) {
  433. /*
  434. * If the section is biggest section in the zone, it need
  435. * shrink zone->spanned_pages.
  436. * In this case, we find second biggest valid mem_section for
  437. * shrinking zone.
  438. */
  439. pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
  440. start_pfn);
  441. if (pfn)
  442. zone->spanned_pages = pfn - zone_start_pfn + 1;
  443. }
  444. /*
  445. * The section is not biggest or smallest mem_section in the zone, it
  446. * only creates a hole in the zone. So in this case, we need not
  447. * change the zone. But perhaps, the zone has only hole data. Thus
  448. * it check the zone has only hole or not.
  449. */
  450. pfn = zone_start_pfn;
  451. for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
  452. ms = __pfn_to_section(pfn);
  453. if (unlikely(!valid_section(ms)))
  454. continue;
  455. if (page_zone(pfn_to_page(pfn)) != zone)
  456. continue;
  457. /* If the section is current section, it continues the loop */
  458. if (start_pfn == pfn)
  459. continue;
  460. /* If we find valid section, we have nothing to do */
  461. zone_span_writeunlock(zone);
  462. return;
  463. }
  464. /* The zone has no valid section */
  465. zone->zone_start_pfn = 0;
  466. zone->spanned_pages = 0;
  467. zone_span_writeunlock(zone);
  468. }
  469. static void shrink_pgdat_span(struct pglist_data *pgdat,
  470. unsigned long start_pfn, unsigned long end_pfn)
  471. {
  472. unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
  473. unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
  474. unsigned long pgdat_end_pfn = p;
  475. unsigned long pfn;
  476. struct mem_section *ms;
  477. int nid = pgdat->node_id;
  478. if (pgdat_start_pfn == start_pfn) {
  479. /*
  480. * If the section is smallest section in the pgdat, it need
  481. * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
  482. * In this case, we find second smallest valid mem_section
  483. * for shrinking zone.
  484. */
  485. pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
  486. pgdat_end_pfn);
  487. if (pfn) {
  488. pgdat->node_start_pfn = pfn;
  489. pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
  490. }
  491. } else if (pgdat_end_pfn == end_pfn) {
  492. /*
  493. * If the section is biggest section in the pgdat, it need
  494. * shrink pgdat->node_spanned_pages.
  495. * In this case, we find second biggest valid mem_section for
  496. * shrinking zone.
  497. */
  498. pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
  499. start_pfn);
  500. if (pfn)
  501. pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
  502. }
  503. /*
  504. * If the section is not biggest or smallest mem_section in the pgdat,
  505. * it only creates a hole in the pgdat. So in this case, we need not
  506. * change the pgdat.
  507. * But perhaps, the pgdat has only hole data. Thus it check the pgdat
  508. * has only hole or not.
  509. */
  510. pfn = pgdat_start_pfn;
  511. for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
  512. ms = __pfn_to_section(pfn);
  513. if (unlikely(!valid_section(ms)))
  514. continue;
  515. if (pfn_to_nid(pfn) != nid)
  516. continue;
  517. /* If the section is current section, it continues the loop */
  518. if (start_pfn == pfn)
  519. continue;
  520. /* If we find valid section, we have nothing to do */
  521. return;
  522. }
  523. /* The pgdat has no valid section */
  524. pgdat->node_start_pfn = 0;
  525. pgdat->node_spanned_pages = 0;
  526. }
  527. static void __remove_zone(struct zone *zone, unsigned long start_pfn)
  528. {
  529. struct pglist_data *pgdat = zone->zone_pgdat;
  530. int nr_pages = PAGES_PER_SECTION;
  531. int zone_type;
  532. unsigned long flags;
  533. zone_type = zone - pgdat->node_zones;
  534. pgdat_resize_lock(zone->zone_pgdat, &flags);
  535. shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
  536. shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
  537. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  538. }
  539. static int __remove_section(struct zone *zone, struct mem_section *ms)
  540. {
  541. unsigned long start_pfn;
  542. int scn_nr;
  543. int ret = -EINVAL;
  544. if (!valid_section(ms))
  545. return ret;
  546. ret = unregister_memory_section(ms);
  547. if (ret)
  548. return ret;
  549. scn_nr = __section_nr(ms);
  550. start_pfn = section_nr_to_pfn(scn_nr);
  551. __remove_zone(zone, start_pfn);
  552. sparse_remove_one_section(zone, ms);
  553. return 0;
  554. }
  555. /**
  556. * __remove_pages() - remove sections of pages from a zone
  557. * @zone: zone from which pages need to be removed
  558. * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
  559. * @nr_pages: number of pages to remove (must be multiple of section size)
  560. *
  561. * Generic helper function to remove section mappings and sysfs entries
  562. * for the section of the memory we are removing. Caller needs to make
  563. * sure that pages are marked reserved and zones are adjust properly by
  564. * calling offline_pages().
  565. */
  566. int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
  567. unsigned long nr_pages)
  568. {
  569. unsigned long i;
  570. int sections_to_remove;
  571. resource_size_t start, size;
  572. int ret = 0;
  573. /*
  574. * We can only remove entire sections
  575. */
  576. BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
  577. BUG_ON(nr_pages % PAGES_PER_SECTION);
  578. start = phys_start_pfn << PAGE_SHIFT;
  579. size = nr_pages * PAGE_SIZE;
  580. ret = release_mem_region_adjustable(&iomem_resource, start, size);
  581. if (ret) {
  582. resource_size_t endres = start + size - 1;
  583. pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
  584. &start, &endres, ret);
  585. }
  586. sections_to_remove = nr_pages / PAGES_PER_SECTION;
  587. for (i = 0; i < sections_to_remove; i++) {
  588. unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
  589. ret = __remove_section(zone, __pfn_to_section(pfn));
  590. if (ret)
  591. break;
  592. }
  593. return ret;
  594. }
  595. EXPORT_SYMBOL_GPL(__remove_pages);
  596. #endif /* CONFIG_MEMORY_HOTREMOVE */
  597. int set_online_page_callback(online_page_callback_t callback)
  598. {
  599. int rc = -EINVAL;
  600. lock_memory_hotplug();
  601. if (online_page_callback == generic_online_page) {
  602. online_page_callback = callback;
  603. rc = 0;
  604. }
  605. unlock_memory_hotplug();
  606. return rc;
  607. }
  608. EXPORT_SYMBOL_GPL(set_online_page_callback);
  609. int restore_online_page_callback(online_page_callback_t callback)
  610. {
  611. int rc = -EINVAL;
  612. lock_memory_hotplug();
  613. if (online_page_callback == callback) {
  614. online_page_callback = generic_online_page;
  615. rc = 0;
  616. }
  617. unlock_memory_hotplug();
  618. return rc;
  619. }
  620. EXPORT_SYMBOL_GPL(restore_online_page_callback);
  621. void __online_page_set_limits(struct page *page)
  622. {
  623. }
  624. EXPORT_SYMBOL_GPL(__online_page_set_limits);
  625. void __online_page_increment_counters(struct page *page)
  626. {
  627. adjust_managed_page_count(page, 1);
  628. }
  629. EXPORT_SYMBOL_GPL(__online_page_increment_counters);
  630. void __online_page_free(struct page *page)
  631. {
  632. __free_reserved_page(page);
  633. }
  634. EXPORT_SYMBOL_GPL(__online_page_free);
  635. static void generic_online_page(struct page *page)
  636. {
  637. __online_page_set_limits(page);
  638. __online_page_increment_counters(page);
  639. __online_page_free(page);
  640. }
  641. static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
  642. void *arg)
  643. {
  644. unsigned long i;
  645. unsigned long onlined_pages = *(unsigned long *)arg;
  646. struct page *page;
  647. if (PageReserved(pfn_to_page(start_pfn)))
  648. for (i = 0; i < nr_pages; i++) {
  649. page = pfn_to_page(start_pfn + i);
  650. (*online_page_callback)(page);
  651. onlined_pages++;
  652. }
  653. *(unsigned long *)arg = onlined_pages;
  654. return 0;
  655. }
  656. #ifdef CONFIG_MOVABLE_NODE
  657. /*
  658. * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
  659. * normal memory.
  660. */
  661. static bool can_online_high_movable(struct zone *zone)
  662. {
  663. return true;
  664. }
  665. #else /* CONFIG_MOVABLE_NODE */
  666. /* ensure every online node has NORMAL memory */
  667. static bool can_online_high_movable(struct zone *zone)
  668. {
  669. return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  670. }
  671. #endif /* CONFIG_MOVABLE_NODE */
  672. /* check which state of node_states will be changed when online memory */
  673. static void node_states_check_changes_online(unsigned long nr_pages,
  674. struct zone *zone, struct memory_notify *arg)
  675. {
  676. int nid = zone_to_nid(zone);
  677. enum zone_type zone_last = ZONE_NORMAL;
  678. /*
  679. * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
  680. * contains nodes which have zones of 0...ZONE_NORMAL,
  681. * set zone_last to ZONE_NORMAL.
  682. *
  683. * If we don't have HIGHMEM nor movable node,
  684. * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
  685. * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
  686. */
  687. if (N_MEMORY == N_NORMAL_MEMORY)
  688. zone_last = ZONE_MOVABLE;
  689. /*
  690. * if the memory to be online is in a zone of 0...zone_last, and
  691. * the zones of 0...zone_last don't have memory before online, we will
  692. * need to set the node to node_states[N_NORMAL_MEMORY] after
  693. * the memory is online.
  694. */
  695. if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
  696. arg->status_change_nid_normal = nid;
  697. else
  698. arg->status_change_nid_normal = -1;
  699. #ifdef CONFIG_HIGHMEM
  700. /*
  701. * If we have movable node, node_states[N_HIGH_MEMORY]
  702. * contains nodes which have zones of 0...ZONE_HIGHMEM,
  703. * set zone_last to ZONE_HIGHMEM.
  704. *
  705. * If we don't have movable node, node_states[N_NORMAL_MEMORY]
  706. * contains nodes which have zones of 0...ZONE_MOVABLE,
  707. * set zone_last to ZONE_MOVABLE.
  708. */
  709. zone_last = ZONE_HIGHMEM;
  710. if (N_MEMORY == N_HIGH_MEMORY)
  711. zone_last = ZONE_MOVABLE;
  712. if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
  713. arg->status_change_nid_high = nid;
  714. else
  715. arg->status_change_nid_high = -1;
  716. #else
  717. arg->status_change_nid_high = arg->status_change_nid_normal;
  718. #endif
  719. /*
  720. * if the node don't have memory befor online, we will need to
  721. * set the node to node_states[N_MEMORY] after the memory
  722. * is online.
  723. */
  724. if (!node_state(nid, N_MEMORY))
  725. arg->status_change_nid = nid;
  726. else
  727. arg->status_change_nid = -1;
  728. }
  729. static void node_states_set_node(int node, struct memory_notify *arg)
  730. {
  731. if (arg->status_change_nid_normal >= 0)
  732. node_set_state(node, N_NORMAL_MEMORY);
  733. if (arg->status_change_nid_high >= 0)
  734. node_set_state(node, N_HIGH_MEMORY);
  735. node_set_state(node, N_MEMORY);
  736. }
  737. int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
  738. {
  739. unsigned long flags;
  740. unsigned long onlined_pages = 0;
  741. struct zone *zone;
  742. int need_zonelists_rebuild = 0;
  743. int nid;
  744. int ret;
  745. struct memory_notify arg;
  746. lock_memory_hotplug();
  747. /*
  748. * This doesn't need a lock to do pfn_to_page().
  749. * The section can't be removed here because of the
  750. * memory_block->state_mutex.
  751. */
  752. zone = page_zone(pfn_to_page(pfn));
  753. if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) &&
  754. !can_online_high_movable(zone)) {
  755. unlock_memory_hotplug();
  756. return -EINVAL;
  757. }
  758. if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) {
  759. if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) {
  760. unlock_memory_hotplug();
  761. return -EINVAL;
  762. }
  763. }
  764. if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) {
  765. if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) {
  766. unlock_memory_hotplug();
  767. return -EINVAL;
  768. }
  769. }
  770. /* Previous code may changed the zone of the pfn range */
  771. zone = page_zone(pfn_to_page(pfn));
  772. arg.start_pfn = pfn;
  773. arg.nr_pages = nr_pages;
  774. node_states_check_changes_online(nr_pages, zone, &arg);
  775. nid = pfn_to_nid(pfn);
  776. ret = memory_notify(MEM_GOING_ONLINE, &arg);
  777. ret = notifier_to_errno(ret);
  778. if (ret) {
  779. memory_notify(MEM_CANCEL_ONLINE, &arg);
  780. unlock_memory_hotplug();
  781. return ret;
  782. }
  783. /*
  784. * If this zone is not populated, then it is not in zonelist.
  785. * This means the page allocator ignores this zone.
  786. * So, zonelist must be updated after online.
  787. */
  788. mutex_lock(&zonelists_mutex);
  789. if (!populated_zone(zone)) {
  790. need_zonelists_rebuild = 1;
  791. build_all_zonelists(NULL, zone);
  792. }
  793. ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
  794. online_pages_range);
  795. if (ret) {
  796. if (need_zonelists_rebuild)
  797. zone_pcp_reset(zone);
  798. mutex_unlock(&zonelists_mutex);
  799. printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
  800. (unsigned long long) pfn << PAGE_SHIFT,
  801. (((unsigned long long) pfn + nr_pages)
  802. << PAGE_SHIFT) - 1);
  803. memory_notify(MEM_CANCEL_ONLINE, &arg);
  804. unlock_memory_hotplug();
  805. return ret;
  806. }
  807. zone->present_pages += onlined_pages;
  808. pgdat_resize_lock(zone->zone_pgdat, &flags);
  809. zone->zone_pgdat->node_present_pages += onlined_pages;
  810. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  811. if (onlined_pages) {
  812. node_states_set_node(zone_to_nid(zone), &arg);
  813. if (need_zonelists_rebuild)
  814. build_all_zonelists(NULL, NULL);
  815. else
  816. zone_pcp_update(zone);
  817. }
  818. mutex_unlock(&zonelists_mutex);
  819. init_per_zone_wmark_min();
  820. if (onlined_pages)
  821. kswapd_run(zone_to_nid(zone));
  822. vm_total_pages = nr_free_pagecache_pages();
  823. writeback_set_ratelimit();
  824. if (onlined_pages)
  825. memory_notify(MEM_ONLINE, &arg);
  826. unlock_memory_hotplug();
  827. return 0;
  828. }
  829. #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
  830. /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
  831. static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
  832. {
  833. struct pglist_data *pgdat;
  834. unsigned long zones_size[MAX_NR_ZONES] = {0};
  835. unsigned long zholes_size[MAX_NR_ZONES] = {0};
  836. unsigned long start_pfn = start >> PAGE_SHIFT;
  837. pgdat = NODE_DATA(nid);
  838. if (!pgdat) {
  839. pgdat = arch_alloc_nodedata(nid);
  840. if (!pgdat)
  841. return NULL;
  842. arch_refresh_nodedata(nid, pgdat);
  843. }
  844. /* we can use NODE_DATA(nid) from here */
  845. /* init node's zones as empty zones, we don't have any present pages.*/
  846. free_area_init_node(nid, zones_size, start_pfn, zholes_size);
  847. /*
  848. * The node we allocated has no zone fallback lists. For avoiding
  849. * to access not-initialized zonelist, build here.
  850. */
  851. mutex_lock(&zonelists_mutex);
  852. build_all_zonelists(pgdat, NULL);
  853. mutex_unlock(&zonelists_mutex);
  854. return pgdat;
  855. }
  856. static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
  857. {
  858. arch_refresh_nodedata(nid, NULL);
  859. arch_free_nodedata(pgdat);
  860. return;
  861. }
  862. /**
  863. * try_online_node - online a node if offlined
  864. *
  865. * called by cpu_up() to online a node without onlined memory.
  866. */
  867. int try_online_node(int nid)
  868. {
  869. pg_data_t *pgdat;
  870. int ret;
  871. if (node_online(nid))
  872. return 0;
  873. lock_memory_hotplug();
  874. pgdat = hotadd_new_pgdat(nid, 0);
  875. if (!pgdat) {
  876. pr_err("Cannot online node %d due to NULL pgdat\n", nid);
  877. ret = -ENOMEM;
  878. goto out;
  879. }
  880. node_set_online(nid);
  881. ret = register_one_node(nid);
  882. BUG_ON(ret);
  883. if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
  884. mutex_lock(&zonelists_mutex);
  885. build_all_zonelists(NULL, NULL);
  886. mutex_unlock(&zonelists_mutex);
  887. }
  888. out:
  889. unlock_memory_hotplug();
  890. return ret;
  891. }
  892. static int check_hotplug_memory_range(u64 start, u64 size)
  893. {
  894. u64 start_pfn = start >> PAGE_SHIFT;
  895. u64 nr_pages = size >> PAGE_SHIFT;
  896. /* Memory range must be aligned with section */
  897. if ((start_pfn & ~PAGE_SECTION_MASK) ||
  898. (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
  899. pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
  900. (unsigned long long)start,
  901. (unsigned long long)size);
  902. return -EINVAL;
  903. }
  904. return 0;
  905. }
  906. /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
  907. int __ref add_memory(int nid, u64 start, u64 size)
  908. {
  909. pg_data_t *pgdat = NULL;
  910. bool new_pgdat;
  911. bool new_node;
  912. struct resource *res;
  913. int ret;
  914. ret = check_hotplug_memory_range(start, size);
  915. if (ret)
  916. return ret;
  917. lock_memory_hotplug();
  918. res = register_memory_resource(start, size);
  919. ret = -EEXIST;
  920. if (!res)
  921. goto out;
  922. { /* Stupid hack to suppress address-never-null warning */
  923. void *p = NODE_DATA(nid);
  924. new_pgdat = !p;
  925. }
  926. new_node = !node_online(nid);
  927. if (new_node) {
  928. pgdat = hotadd_new_pgdat(nid, start);
  929. ret = -ENOMEM;
  930. if (!pgdat)
  931. goto error;
  932. }
  933. /* call arch's memory hotadd */
  934. ret = arch_add_memory(nid, start, size);
  935. if (ret < 0)
  936. goto error;
  937. /* we online node here. we can't roll back from here. */
  938. node_set_online(nid);
  939. if (new_node) {
  940. ret = register_one_node(nid);
  941. /*
  942. * If sysfs file of new node can't create, cpu on the node
  943. * can't be hot-added. There is no rollback way now.
  944. * So, check by BUG_ON() to catch it reluctantly..
  945. */
  946. BUG_ON(ret);
  947. }
  948. /* create new memmap entry */
  949. firmware_map_add_hotplug(start, start + size, "System RAM");
  950. goto out;
  951. error:
  952. /* rollback pgdat allocation and others */
  953. if (new_pgdat)
  954. rollback_node_hotadd(nid, pgdat);
  955. release_memory_resource(res);
  956. out:
  957. unlock_memory_hotplug();
  958. return ret;
  959. }
  960. EXPORT_SYMBOL_GPL(add_memory);
  961. #ifdef CONFIG_MEMORY_HOTREMOVE
  962. /*
  963. * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
  964. * set and the size of the free page is given by page_order(). Using this,
  965. * the function determines if the pageblock contains only free pages.
  966. * Due to buddy contraints, a free page at least the size of a pageblock will
  967. * be located at the start of the pageblock
  968. */
  969. static inline int pageblock_free(struct page *page)
  970. {
  971. return PageBuddy(page) && page_order(page) >= pageblock_order;
  972. }
  973. /* Return the start of the next active pageblock after a given page */
  974. static struct page *next_active_pageblock(struct page *page)
  975. {
  976. /* Ensure the starting page is pageblock-aligned */
  977. BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));
  978. /* If the entire pageblock is free, move to the end of free page */
  979. if (pageblock_free(page)) {
  980. int order;
  981. /* be careful. we don't have locks, page_order can be changed.*/
  982. order = page_order(page);
  983. if ((order < MAX_ORDER) && (order >= pageblock_order))
  984. return page + (1 << order);
  985. }
  986. return page + pageblock_nr_pages;
  987. }
  988. /* Checks if this range of memory is likely to be hot-removable. */
  989. int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
  990. {
  991. struct page *page = pfn_to_page(start_pfn);
  992. struct page *end_page = page + nr_pages;
  993. /* Check the starting page of each pageblock within the range */
  994. for (; page < end_page; page = next_active_pageblock(page)) {
  995. if (!is_pageblock_removable_nolock(page))
  996. return 0;
  997. cond_resched();
  998. }
  999. /* All pageblocks in the memory block are likely to be hot-removable */
  1000. return 1;
  1001. }
  1002. /*
  1003. * Confirm all pages in a range [start, end) is belongs to the same zone.
  1004. */
  1005. static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
  1006. {
  1007. unsigned long pfn;
  1008. struct zone *zone = NULL;
  1009. struct page *page;
  1010. int i;
  1011. for (pfn = start_pfn;
  1012. pfn < end_pfn;
  1013. pfn += MAX_ORDER_NR_PAGES) {
  1014. i = 0;
  1015. /* This is just a CONFIG_HOLES_IN_ZONE check.*/
  1016. while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i))
  1017. i++;
  1018. if (i == MAX_ORDER_NR_PAGES)
  1019. continue;
  1020. page = pfn_to_page(pfn + i);
  1021. if (zone && page_zone(page) != zone)
  1022. return 0;
  1023. zone = page_zone(page);
  1024. }
  1025. return 1;
  1026. }
  1027. /*
  1028. * Scan pfn range [start,end) to find movable/migratable pages (LRU pages
  1029. * and hugepages). We scan pfn because it's much easier than scanning over
  1030. * linked list. This function returns the pfn of the first found movable
  1031. * page if it's found, otherwise 0.
  1032. */
  1033. static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
  1034. {
  1035. unsigned long pfn;
  1036. struct page *page;
  1037. for (pfn = start; pfn < end; pfn++) {
  1038. if (pfn_valid(pfn)) {
  1039. page = pfn_to_page(pfn);
  1040. if (PageLRU(page))
  1041. return pfn;
  1042. if (PageHuge(page)) {
  1043. if (is_hugepage_active(page))
  1044. return pfn;
  1045. else
  1046. pfn = round_up(pfn + 1,
  1047. 1 << compound_order(page)) - 1;
  1048. }
  1049. }
  1050. }
  1051. return 0;
  1052. }
  1053. #define NR_OFFLINE_AT_ONCE_PAGES (256)
  1054. static int
  1055. do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
  1056. {
  1057. unsigned long pfn;
  1058. struct page *page;
  1059. int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
  1060. int not_managed = 0;
  1061. int ret = 0;
  1062. LIST_HEAD(source);
  1063. for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
  1064. if (!pfn_valid(pfn))
  1065. continue;
  1066. page = pfn_to_page(pfn);
  1067. if (PageHuge(page)) {
  1068. struct page *head = compound_head(page);
  1069. pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
  1070. if (compound_order(head) > PFN_SECTION_SHIFT) {
  1071. ret = -EBUSY;
  1072. break;
  1073. }
  1074. if (isolate_huge_page(page, &source))
  1075. move_pages -= 1 << compound_order(head);
  1076. continue;
  1077. }
  1078. if (!get_page_unless_zero(page))
  1079. continue;
  1080. /*
  1081. * We can skip free pages. And we can only deal with pages on
  1082. * LRU.
  1083. */
  1084. ret = isolate_lru_page(page);
  1085. if (!ret) { /* Success */
  1086. put_page(page);
  1087. list_add_tail(&page->lru, &source);
  1088. move_pages--;
  1089. inc_zone_page_state(page, NR_ISOLATED_ANON +
  1090. page_is_file_cache(page));
  1091. } else {
  1092. #ifdef CONFIG_DEBUG_VM
  1093. printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
  1094. pfn);
  1095. dump_page(page);
  1096. #endif
  1097. put_page(page);
  1098. /* Because we don't have big zone->lock. we should
  1099. check this again here. */
  1100. if (page_count(page)) {
  1101. not_managed++;
  1102. ret = -EBUSY;
  1103. break;
  1104. }
  1105. }
  1106. }
  1107. if (!list_empty(&source)) {
  1108. if (not_managed) {
  1109. putback_movable_pages(&source);
  1110. goto out;
  1111. }
  1112. /*
  1113. * alloc_migrate_target should be improooooved!!
  1114. * migrate_pages returns # of failed pages.
  1115. */
  1116. ret = migrate_pages(&source, alloc_migrate_target, 0,
  1117. MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
  1118. if (ret)
  1119. putback_movable_pages(&source);
  1120. }
  1121. out:
  1122. return ret;
  1123. }
  1124. /*
  1125. * remove from free_area[] and mark all as Reserved.
  1126. */
  1127. static int
  1128. offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
  1129. void *data)
  1130. {
  1131. __offline_isolated_pages(start, start + nr_pages);
  1132. return 0;
  1133. }
  1134. static void
  1135. offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  1136. {
  1137. walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
  1138. offline_isolated_pages_cb);
  1139. }
  1140. /*
  1141. * Check all pages in range, recoreded as memory resource, are isolated.
  1142. */
  1143. static int
  1144. check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
  1145. void *data)
  1146. {
  1147. int ret;
  1148. long offlined = *(long *)data;
  1149. ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
  1150. offlined = nr_pages;
  1151. if (!ret)
  1152. *(long *)data += offlined;
  1153. return ret;
  1154. }
  1155. static long
  1156. check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
  1157. {
  1158. long offlined = 0;
  1159. int ret;
  1160. ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
  1161. check_pages_isolated_cb);
  1162. if (ret < 0)
  1163. offlined = (long)ret;
  1164. return offlined;
  1165. }
  1166. #ifdef CONFIG_MOVABLE_NODE
  1167. /*
  1168. * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
  1169. * normal memory.
  1170. */
  1171. static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
  1172. {
  1173. return true;
  1174. }
  1175. #else /* CONFIG_MOVABLE_NODE */
  1176. /* ensure the node has NORMAL memory if it is still online */
  1177. static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
  1178. {
  1179. struct pglist_data *pgdat = zone->zone_pgdat;
  1180. unsigned long present_pages = 0;
  1181. enum zone_type zt;
  1182. for (zt = 0; zt <= ZONE_NORMAL; zt++)
  1183. present_pages += pgdat->node_zones[zt].present_pages;
  1184. if (present_pages > nr_pages)
  1185. return true;
  1186. present_pages = 0;
  1187. for (; zt <= ZONE_MOVABLE; zt++)
  1188. present_pages += pgdat->node_zones[zt].present_pages;
  1189. /*
  1190. * we can't offline the last normal memory until all
  1191. * higher memory is offlined.
  1192. */
  1193. return present_pages == 0;
  1194. }
  1195. #endif /* CONFIG_MOVABLE_NODE */
  1196. static int __init cmdline_parse_movable_node(char *p)
  1197. {
  1198. #ifdef CONFIG_MOVABLE_NODE
  1199. /*
  1200. * Memory used by the kernel cannot be hot-removed because Linux
  1201. * cannot migrate the kernel pages. When memory hotplug is
  1202. * enabled, we should prevent memblock from allocating memory
  1203. * for the kernel.
  1204. *
  1205. * ACPI SRAT records all hotpluggable memory ranges. But before
  1206. * SRAT is parsed, we don't know about it.
  1207. *
  1208. * The kernel image is loaded into memory at very early time. We
  1209. * cannot prevent this anyway. So on NUMA system, we set any
  1210. * node the kernel resides in as un-hotpluggable.
  1211. *
  1212. * Since on modern servers, one node could have double-digit
  1213. * gigabytes memory, we can assume the memory around the kernel
  1214. * image is also un-hotpluggable. So before SRAT is parsed, just
  1215. * allocate memory near the kernel image to try the best to keep
  1216. * the kernel away from hotpluggable memory.
  1217. */
  1218. memblock_set_bottom_up(true);
  1219. #else
  1220. pr_warn("movable_node option not supported\n");
  1221. #endif
  1222. return 0;
  1223. }
  1224. early_param("movable_node", cmdline_parse_movable_node);
  1225. /* check which state of node_states will be changed when offline memory */
  1226. static void node_states_check_changes_offline(unsigned long nr_pages,
  1227. struct zone *zone, struct memory_notify *arg)
  1228. {
  1229. struct pglist_data *pgdat = zone->zone_pgdat;
  1230. unsigned long present_pages = 0;
  1231. enum zone_type zt, zone_last = ZONE_NORMAL;
  1232. /*
  1233. * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
  1234. * contains nodes which have zones of 0...ZONE_NORMAL,
  1235. * set zone_last to ZONE_NORMAL.
  1236. *
  1237. * If we don't have HIGHMEM nor movable node,
  1238. * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
  1239. * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
  1240. */
  1241. if (N_MEMORY == N_NORMAL_MEMORY)
  1242. zone_last = ZONE_MOVABLE;
  1243. /*
  1244. * check whether node_states[N_NORMAL_MEMORY] will be changed.
  1245. * If the memory to be offline is in a zone of 0...zone_last,
  1246. * and it is the last present memory, 0...zone_last will
  1247. * become empty after offline , thus we can determind we will
  1248. * need to clear the node from node_states[N_NORMAL_MEMORY].
  1249. */
  1250. for (zt = 0; zt <= zone_last; zt++)
  1251. present_pages += pgdat->node_zones[zt].present_pages;
  1252. if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
  1253. arg->status_change_nid_normal = zone_to_nid(zone);
  1254. else
  1255. arg->status_change_nid_normal = -1;
  1256. #ifdef CONFIG_HIGHMEM
  1257. /*
  1258. * If we have movable node, node_states[N_HIGH_MEMORY]
  1259. * contains nodes which have zones of 0...ZONE_HIGHMEM,
  1260. * set zone_last to ZONE_HIGHMEM.
  1261. *
  1262. * If we don't have movable node, node_states[N_NORMAL_MEMORY]
  1263. * contains nodes which have zones of 0...ZONE_MOVABLE,
  1264. * set zone_last to ZONE_MOVABLE.
  1265. */
  1266. zone_last = ZONE_HIGHMEM;
  1267. if (N_MEMORY == N_HIGH_MEMORY)
  1268. zone_last = ZONE_MOVABLE;
  1269. for (; zt <= zone_last; zt++)
  1270. present_pages += pgdat->node_zones[zt].present_pages;
  1271. if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
  1272. arg->status_change_nid_high = zone_to_nid(zone);
  1273. else
  1274. arg->status_change_nid_high = -1;
  1275. #else
  1276. arg->status_change_nid_high = arg->status_change_nid_normal;
  1277. #endif
  1278. /*
  1279. * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
  1280. */
  1281. zone_last = ZONE_MOVABLE;
  1282. /*
  1283. * check whether node_states[N_HIGH_MEMORY] will be changed
  1284. * If we try to offline the last present @nr_pages from the node,
  1285. * we can determind we will need to clear the node from
  1286. * node_states[N_HIGH_MEMORY].
  1287. */
  1288. for (; zt <= zone_last; zt++)
  1289. present_pages += pgdat->node_zones[zt].present_pages;
  1290. if (nr_pages >= present_pages)
  1291. arg->status_change_nid = zone_to_nid(zone);
  1292. else
  1293. arg->status_change_nid = -1;
  1294. }
  1295. static void node_states_clear_node(int node, struct memory_notify *arg)
  1296. {
  1297. if (arg->status_change_nid_normal >= 0)
  1298. node_clear_state(node, N_NORMAL_MEMORY);
  1299. if ((N_MEMORY != N_NORMAL_MEMORY) &&
  1300. (arg->status_change_nid_high >= 0))
  1301. node_clear_state(node, N_HIGH_MEMORY);
  1302. if ((N_MEMORY != N_HIGH_MEMORY) &&
  1303. (arg->status_change_nid >= 0))
  1304. node_clear_state(node, N_MEMORY);
  1305. }
  1306. static int __ref __offline_pages(unsigned long start_pfn,
  1307. unsigned long end_pfn, unsigned long timeout)
  1308. {
  1309. unsigned long pfn, nr_pages, expire;
  1310. long offlined_pages;
  1311. int ret, drain, retry_max, node;
  1312. unsigned long flags;
  1313. struct zone *zone;
  1314. struct memory_notify arg;
  1315. /* at least, alignment against pageblock is necessary */
  1316. if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
  1317. return -EINVAL;
  1318. if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
  1319. return -EINVAL;
  1320. /* This makes hotplug much easier...and readable.
  1321. we assume this for now. .*/
  1322. if (!test_pages_in_a_zone(start_pfn, end_pfn))
  1323. return -EINVAL;
  1324. lock_memory_hotplug();
  1325. zone = page_zone(pfn_to_page(start_pfn));
  1326. node = zone_to_nid(zone);
  1327. nr_pages = end_pfn - start_pfn;
  1328. ret = -EINVAL;
  1329. if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
  1330. goto out;
  1331. /* set above range as isolated */
  1332. ret = start_isolate_page_range(start_pfn, end_pfn,
  1333. MIGRATE_MOVABLE, true);
  1334. if (ret)
  1335. goto out;
  1336. arg.start_pfn = start_pfn;
  1337. arg.nr_pages = nr_pages;
  1338. node_states_check_changes_offline(nr_pages, zone, &arg);
  1339. ret = memory_notify(MEM_GOING_OFFLINE, &arg);
  1340. ret = notifier_to_errno(ret);
  1341. if (ret)
  1342. goto failed_removal;
  1343. pfn = start_pfn;
  1344. expire = jiffies + timeout;
  1345. drain = 0;
  1346. retry_max = 5;
  1347. repeat:
  1348. /* start memory hot removal */
  1349. ret = -EAGAIN;
  1350. if (time_after(jiffies, expire))
  1351. goto failed_removal;
  1352. ret = -EINTR;
  1353. if (signal_pending(current))
  1354. goto failed_removal;
  1355. ret = 0;
  1356. if (drain) {
  1357. lru_add_drain_all();
  1358. cond_resched();
  1359. drain_all_pages();
  1360. }
  1361. pfn = scan_movable_pages(start_pfn, end_pfn);
  1362. if (pfn) { /* We have movable pages */
  1363. ret = do_migrate_range(pfn, end_pfn);
  1364. if (!ret) {
  1365. drain = 1;
  1366. goto repeat;
  1367. } else {
  1368. if (ret < 0)
  1369. if (--retry_max == 0)
  1370. goto failed_removal;
  1371. yield();
  1372. drain = 1;
  1373. goto repeat;
  1374. }
  1375. }
  1376. /* drain all zone's lru pagevec, this is asynchronous... */
  1377. lru_add_drain_all();
  1378. yield();
  1379. /* drain pcp pages, this is synchronous. */
  1380. drain_all_pages();
  1381. /*
  1382. * dissolve free hugepages in the memory block before doing offlining
  1383. * actually in order to make hugetlbfs's object counting consistent.
  1384. */
  1385. dissolve_free_huge_pages(start_pfn, end_pfn);
  1386. /* check again */
  1387. offlined_pages = check_pages_isolated(start_pfn, end_pfn);
  1388. if (offlined_pages < 0) {
  1389. ret = -EBUSY;
  1390. goto failed_removal;
  1391. }
  1392. printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
  1393. /* Ok, all of our target is isolated.
  1394. We cannot do rollback at this point. */
  1395. offline_isolated_pages(start_pfn, end_pfn);
  1396. /* reset pagetype flags and makes migrate type to be MOVABLE */
  1397. undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  1398. /* removal success */
  1399. adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
  1400. zone->present_pages -= offlined_pages;
  1401. pgdat_resize_lock(zone->zone_pgdat, &flags);
  1402. zone->zone_pgdat->node_present_pages -= offlined_pages;
  1403. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  1404. init_per_zone_wmark_min();
  1405. if (!populated_zone(zone)) {
  1406. zone_pcp_reset(zone);
  1407. mutex_lock(&zonelists_mutex);
  1408. build_all_zonelists(NULL, NULL);
  1409. mutex_unlock(&zonelists_mutex);
  1410. } else
  1411. zone_pcp_update(zone);
  1412. node_states_clear_node(node, &arg);
  1413. if (arg.status_change_nid >= 0)
  1414. kswapd_stop(node);
  1415. vm_total_pages = nr_free_pagecache_pages();
  1416. writeback_set_ratelimit();
  1417. memory_notify(MEM_OFFLINE, &arg);
  1418. unlock_memory_hotplug();
  1419. return 0;
  1420. failed_removal:
  1421. printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
  1422. (unsigned long long) start_pfn << PAGE_SHIFT,
  1423. ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
  1424. memory_notify(MEM_CANCEL_OFFLINE, &arg);
  1425. /* pushback to free area */
  1426. undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  1427. out:
  1428. unlock_memory_hotplug();
  1429. return ret;
  1430. }
  1431. int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
  1432. {
  1433. return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
  1434. }
  1435. #endif /* CONFIG_MEMORY_HOTREMOVE */
  1436. /**
  1437. * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
  1438. * @start_pfn: start pfn of the memory range
  1439. * @end_pfn: end pfn of the memory range
  1440. * @arg: argument passed to func
  1441. * @func: callback for each memory section walked
  1442. *
  1443. * This function walks through all present mem sections in range
  1444. * [start_pfn, end_pfn) and call func on each mem section.
  1445. *
  1446. * Returns the return value of func.
  1447. */
  1448. int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
  1449. void *arg, int (*func)(struct memory_block *, void *))
  1450. {
  1451. struct memory_block *mem = NULL;
  1452. struct mem_section *section;
  1453. unsigned long pfn, section_nr;
  1454. int ret;
  1455. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  1456. section_nr = pfn_to_section_nr(pfn);
  1457. if (!present_section_nr(section_nr))
  1458. continue;
  1459. section = __nr_to_section(section_nr);
  1460. /* same memblock? */
  1461. if (mem)
  1462. if ((section_nr >= mem->start_section_nr) &&
  1463. (section_nr <= mem->end_section_nr))
  1464. continue;
  1465. mem = find_memory_block_hinted(section, mem);
  1466. if (!mem)
  1467. continue;
  1468. ret = func(mem, arg);
  1469. if (ret) {
  1470. kobject_put(&mem->dev.kobj);
  1471. return ret;
  1472. }
  1473. }
  1474. if (mem)
  1475. kobject_put(&mem->dev.kobj);
  1476. return 0;
  1477. }
  1478. #ifdef CONFIG_MEMORY_HOTREMOVE
  1479. static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
  1480. {
  1481. int ret = !is_memblock_offlined(mem);
  1482. if (unlikely(ret)) {
  1483. phys_addr_t beginpa, endpa;
  1484. beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
  1485. endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
  1486. pr_warn("removing memory fails, because memory "
  1487. "[%pa-%pa] is onlined\n",
  1488. &beginpa, &endpa);
  1489. }
  1490. return ret;
  1491. }
  1492. static int check_cpu_on_node(pg_data_t *pgdat)
  1493. {
  1494. int cpu;
  1495. for_each_present_cpu(cpu) {
  1496. if (cpu_to_node(cpu) == pgdat->node_id)
  1497. /*
  1498. * the cpu on this node isn't removed, and we can't
  1499. * offline this node.
  1500. */
  1501. return -EBUSY;
  1502. }
  1503. return 0;
  1504. }
  1505. static void unmap_cpu_on_node(pg_data_t *pgdat)
  1506. {
  1507. #ifdef CONFIG_ACPI_NUMA
  1508. int cpu;
  1509. for_each_possible_cpu(cpu)
  1510. if (cpu_to_node(cpu) == pgdat->node_id)
  1511. numa_clear_node(cpu);
  1512. #endif
  1513. }
  1514. static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
  1515. {
  1516. int ret;
  1517. ret = check_cpu_on_node(pgdat);
  1518. if (ret)
  1519. return ret;
  1520. /*
  1521. * the node will be offlined when we come here, so we can clear
  1522. * the cpu_to_node() now.
  1523. */
  1524. unmap_cpu_on_node(pgdat);
  1525. return 0;
  1526. }
  1527. /**
  1528. * try_offline_node
  1529. *
  1530. * Offline a node if all memory sections and cpus of the node are removed.
  1531. *
  1532. * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
  1533. * and online/offline operations before this call.
  1534. */
  1535. void try_offline_node(int nid)
  1536. {
  1537. pg_data_t *pgdat = NODE_DATA(nid);
  1538. unsigned long start_pfn = pgdat->node_start_pfn;
  1539. unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
  1540. unsigned long pfn;
  1541. struct page *pgdat_page = virt_to_page(pgdat);
  1542. int i;
  1543. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  1544. unsigned long section_nr = pfn_to_section_nr(pfn);
  1545. if (!present_section_nr(section_nr))
  1546. continue;
  1547. if (pfn_to_nid(pfn) != nid)
  1548. continue;
  1549. /*
  1550. * some memory sections of this node are not removed, and we
  1551. * can't offline node now.
  1552. */
  1553. return;
  1554. }
  1555. if (check_and_unmap_cpu_on_node(pgdat))
  1556. return;
  1557. /*
  1558. * all memory/cpu of this node are removed, we can offline this
  1559. * node now.
  1560. */
  1561. node_set_offline(nid);
  1562. unregister_one_node(nid);
  1563. if (!PageSlab(pgdat_page) && !PageCompound(pgdat_page))
  1564. /* node data is allocated from boot memory */
  1565. return;
  1566. /* free waittable in each zone */
  1567. for (i = 0; i < MAX_NR_ZONES; i++) {
  1568. struct zone *zone = pgdat->node_zones + i;
  1569. /*
  1570. * wait_table may be allocated from boot memory,
  1571. * here only free if it's allocated by vmalloc.
  1572. */
  1573. if (is_vmalloc_addr(zone->wait_table))
  1574. vfree(zone->wait_table);
  1575. }
  1576. /*
  1577. * Since there is no way to guarentee the address of pgdat/zone is not
  1578. * on stack of any kernel threads or used by other kernel objects
  1579. * without reference counting or other symchronizing method, do not
  1580. * reset node_data and free pgdat here. Just reset it to 0 and reuse
  1581. * the memory when the node is online again.
  1582. */
  1583. memset(pgdat, 0, sizeof(*pgdat));
  1584. }
  1585. EXPORT_SYMBOL(try_offline_node);
  1586. /**
  1587. * remove_memory
  1588. *
  1589. * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
  1590. * and online/offline operations before this call, as required by
  1591. * try_offline_node().
  1592. */
  1593. void __ref remove_memory(int nid, u64 start, u64 size)
  1594. {
  1595. int ret;
  1596. BUG_ON(check_hotplug_memory_range(start, size));
  1597. lock_memory_hotplug();
  1598. /*
  1599. * All memory blocks must be offlined before removing memory. Check
  1600. * whether all memory blocks in question are offline and trigger a BUG()
  1601. * if this is not the case.
  1602. */
  1603. ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
  1604. check_memblock_offlined_cb);
  1605. if (ret) {
  1606. unlock_memory_hotplug();
  1607. BUG();
  1608. }
  1609. /* remove memmap entry */
  1610. firmware_map_remove(start, start + size, "System RAM");
  1611. arch_remove_memory(start, size);
  1612. try_offline_node(nid);
  1613. unlock_memory_hotplug();
  1614. }
  1615. EXPORT_SYMBOL_GPL(remove_memory);
  1616. #endif /* CONFIG_MEMORY_HOTREMOVE */