memory_hotplug.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898
  1. /*
  2. * linux/mm/memory_hotplug.c
  3. *
  4. * Copyright (C)
  5. */
  6. #include <linux/stddef.h>
  7. #include <linux/mm.h>
  8. #include <linux/swap.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/bootmem.h>
  12. #include <linux/compiler.h>
  13. #include <linux/export.h>
  14. #include <linux/pagevec.h>
  15. #include <linux/writeback.h>
  16. #include <linux/slab.h>
  17. #include <linux/sysctl.h>
  18. #include <linux/cpu.h>
  19. #include <linux/memory.h>
  20. #include <linux/memory_hotplug.h>
  21. #include <linux/highmem.h>
  22. #include <linux/vmalloc.h>
  23. #include <linux/ioport.h>
  24. #include <linux/delay.h>
  25. #include <linux/migrate.h>
  26. #include <linux/page-isolation.h>
  27. #include <linux/pfn.h>
  28. #include <linux/suspend.h>
  29. #include <linux/mm_inline.h>
  30. #include <linux/firmware-map.h>
  31. #include <linux/stop_machine.h>
  32. #include <asm/tlbflush.h>
  33. #include "internal.h"
  34. /*
  35. * online_page_callback contains pointer to current page onlining function.
  36. * Initially it is generic_online_page(). If it is required it could be
  37. * changed by calling set_online_page_callback() for callback registration
  38. * and restore_online_page_callback() for generic callback restore.
  39. */
  40. static void generic_online_page(struct page *page);
  41. static online_page_callback_t online_page_callback = generic_online_page;
  42. DEFINE_MUTEX(mem_hotplug_mutex);
  43. void lock_memory_hotplug(void)
  44. {
  45. mutex_lock(&mem_hotplug_mutex);
  46. /* for exclusive hibernation if CONFIG_HIBERNATION=y */
  47. lock_system_sleep();
  48. }
  49. void unlock_memory_hotplug(void)
  50. {
  51. unlock_system_sleep();
  52. mutex_unlock(&mem_hotplug_mutex);
  53. }
  54. /* add this memory to iomem resource */
  55. static struct resource *register_memory_resource(u64 start, u64 size)
  56. {
  57. struct resource *res;
  58. res = kzalloc(sizeof(struct resource), GFP_KERNEL);
  59. BUG_ON(!res);
  60. res->name = "System RAM";
  61. res->start = start;
  62. res->end = start + size - 1;
  63. res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
  64. if (request_resource(&iomem_resource, res) < 0) {
  65. printk("System RAM resource %pR cannot be added\n", res);
  66. kfree(res);
  67. res = NULL;
  68. }
  69. return res;
  70. }
  71. static void release_memory_resource(struct resource *res)
  72. {
  73. if (!res)
  74. return;
  75. release_resource(res);
  76. kfree(res);
  77. return;
  78. }
  79. #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
  80. void get_page_bootmem(unsigned long info, struct page *page,
  81. unsigned long type)
  82. {
  83. page->lru.next = (struct list_head *) type;
  84. SetPagePrivate(page);
  85. set_page_private(page, info);
  86. atomic_inc(&page->_count);
  87. }
  88. /* reference to __meminit __free_pages_bootmem is valid
  89. * so use __ref to tell modpost not to generate a warning */
  90. void __ref put_page_bootmem(struct page *page)
  91. {
  92. unsigned long type;
  93. static DEFINE_MUTEX(ppb_lock);
  94. type = (unsigned long) page->lru.next;
  95. BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
  96. type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
  97. if (atomic_dec_return(&page->_count) == 1) {
  98. ClearPagePrivate(page);
  99. set_page_private(page, 0);
  100. INIT_LIST_HEAD(&page->lru);
  101. /*
  102. * Please refer to comment for __free_pages_bootmem()
  103. * for why we serialize here.
  104. */
  105. mutex_lock(&ppb_lock);
  106. __free_pages_bootmem(page, 0);
  107. mutex_unlock(&ppb_lock);
  108. totalram_pages++;
  109. }
  110. }
  111. #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
  112. #ifndef CONFIG_SPARSEMEM_VMEMMAP
  113. static void register_page_bootmem_info_section(unsigned long start_pfn)
  114. {
  115. unsigned long *usemap, mapsize, section_nr, i;
  116. struct mem_section *ms;
  117. struct page *page, *memmap;
  118. section_nr = pfn_to_section_nr(start_pfn);
  119. ms = __nr_to_section(section_nr);
  120. /* Get section's memmap address */
  121. memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  122. /*
  123. * Get page for the memmap's phys address
  124. * XXX: need more consideration for sparse_vmemmap...
  125. */
  126. page = virt_to_page(memmap);
  127. mapsize = sizeof(struct page) * PAGES_PER_SECTION;
  128. mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
  129. /* remember memmap's page */
  130. for (i = 0; i < mapsize; i++, page++)
  131. get_page_bootmem(section_nr, page, SECTION_INFO);
  132. usemap = __nr_to_section(section_nr)->pageblock_flags;
  133. page = virt_to_page(usemap);
  134. mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
  135. for (i = 0; i < mapsize; i++, page++)
  136. get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  137. }
  138. #else /* CONFIG_SPARSEMEM_VMEMMAP */
  139. static void register_page_bootmem_info_section(unsigned long start_pfn)
  140. {
  141. unsigned long *usemap, mapsize, section_nr, i;
  142. struct mem_section *ms;
  143. struct page *page, *memmap;
  144. if (!pfn_valid(start_pfn))
  145. return;
  146. section_nr = pfn_to_section_nr(start_pfn);
  147. ms = __nr_to_section(section_nr);
  148. memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  149. register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
  150. usemap = __nr_to_section(section_nr)->pageblock_flags;
  151. page = virt_to_page(usemap);
  152. mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
  153. for (i = 0; i < mapsize; i++, page++)
  154. get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  155. }
  156. #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
  157. void register_page_bootmem_info_node(struct pglist_data *pgdat)
  158. {
  159. unsigned long i, pfn, end_pfn, nr_pages;
  160. int node = pgdat->node_id;
  161. struct page *page;
  162. struct zone *zone;
  163. nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
  164. page = virt_to_page(pgdat);
  165. for (i = 0; i < nr_pages; i++, page++)
  166. get_page_bootmem(node, page, NODE_INFO);
  167. zone = &pgdat->node_zones[0];
  168. for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
  169. if (zone->wait_table) {
  170. nr_pages = zone->wait_table_hash_nr_entries
  171. * sizeof(wait_queue_head_t);
  172. nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
  173. page = virt_to_page(zone->wait_table);
  174. for (i = 0; i < nr_pages; i++, page++)
  175. get_page_bootmem(node, page, NODE_INFO);
  176. }
  177. }
  178. pfn = pgdat->node_start_pfn;
  179. end_pfn = pgdat_end_pfn(pgdat);
  180. /* register_section info */
  181. for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  182. /*
  183. * Some platforms can assign the same pfn to multiple nodes - on
  184. * node0 as well as nodeN. To avoid registering a pfn against
  185. * multiple nodes we check that this pfn does not already
  186. * reside in some other node.
  187. */
  188. if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node))
  189. register_page_bootmem_info_section(pfn);
  190. }
  191. }
  192. #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
  193. static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
  194. unsigned long end_pfn)
  195. {
  196. unsigned long old_zone_end_pfn;
  197. zone_span_writelock(zone);
  198. old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
  199. if (!zone->spanned_pages || start_pfn < zone->zone_start_pfn)
  200. zone->zone_start_pfn = start_pfn;
  201. zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
  202. zone->zone_start_pfn;
  203. zone_span_writeunlock(zone);
  204. }
  205. static void resize_zone(struct zone *zone, unsigned long start_pfn,
  206. unsigned long end_pfn)
  207. {
  208. zone_span_writelock(zone);
  209. if (end_pfn - start_pfn) {
  210. zone->zone_start_pfn = start_pfn;
  211. zone->spanned_pages = end_pfn - start_pfn;
  212. } else {
  213. /*
  214. * make it consist as free_area_init_core(),
  215. * if spanned_pages = 0, then keep start_pfn = 0
  216. */
  217. zone->zone_start_pfn = 0;
  218. zone->spanned_pages = 0;
  219. }
  220. zone_span_writeunlock(zone);
  221. }
  222. static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
  223. unsigned long end_pfn)
  224. {
  225. enum zone_type zid = zone_idx(zone);
  226. int nid = zone->zone_pgdat->node_id;
  227. unsigned long pfn;
  228. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  229. set_page_links(pfn_to_page(pfn), zid, nid, pfn);
  230. }
  231. /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
  232. * alloc_bootmem_node_nopanic() */
  233. static int __ref ensure_zone_is_initialized(struct zone *zone,
  234. unsigned long start_pfn, unsigned long num_pages)
  235. {
  236. if (!zone_is_initialized(zone))
  237. return init_currently_empty_zone(zone, start_pfn, num_pages,
  238. MEMMAP_HOTPLUG);
  239. return 0;
  240. }
  241. static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
  242. unsigned long start_pfn, unsigned long end_pfn)
  243. {
  244. int ret;
  245. unsigned long flags;
  246. unsigned long z1_start_pfn;
  247. ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
  248. if (ret)
  249. return ret;
  250. pgdat_resize_lock(z1->zone_pgdat, &flags);
  251. /* can't move pfns which are higher than @z2 */
  252. if (end_pfn > zone_end_pfn(z2))
  253. goto out_fail;
  254. /* the move out part mast at the left most of @z2 */
  255. if (start_pfn > z2->zone_start_pfn)
  256. goto out_fail;
  257. /* must included/overlap */
  258. if (end_pfn <= z2->zone_start_pfn)
  259. goto out_fail;
  260. /* use start_pfn for z1's start_pfn if z1 is empty */
  261. if (z1->spanned_pages)
  262. z1_start_pfn = z1->zone_start_pfn;
  263. else
  264. z1_start_pfn = start_pfn;
  265. resize_zone(z1, z1_start_pfn, end_pfn);
  266. resize_zone(z2, end_pfn, zone_end_pfn(z2));
  267. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  268. fix_zone_id(z1, start_pfn, end_pfn);
  269. return 0;
  270. out_fail:
  271. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  272. return -1;
  273. }
  274. static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
  275. unsigned long start_pfn, unsigned long end_pfn)
  276. {
  277. int ret;
  278. unsigned long flags;
  279. unsigned long z2_end_pfn;
  280. ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
  281. if (ret)
  282. return ret;
  283. pgdat_resize_lock(z1->zone_pgdat, &flags);
  284. /* can't move pfns which are lower than @z1 */
  285. if (z1->zone_start_pfn > start_pfn)
  286. goto out_fail;
  287. /* the move out part mast at the right most of @z1 */
  288. if (zone_end_pfn(z1) > end_pfn)
  289. goto out_fail;
  290. /* must included/overlap */
  291. if (start_pfn >= zone_end_pfn(z1))
  292. goto out_fail;
  293. /* use end_pfn for z2's end_pfn if z2 is empty */
  294. if (z2->spanned_pages)
  295. z2_end_pfn = zone_end_pfn(z2);
  296. else
  297. z2_end_pfn = end_pfn;
  298. resize_zone(z1, z1->zone_start_pfn, start_pfn);
  299. resize_zone(z2, start_pfn, z2_end_pfn);
  300. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  301. fix_zone_id(z2, start_pfn, end_pfn);
  302. return 0;
  303. out_fail:
  304. pgdat_resize_unlock(z1->zone_pgdat, &flags);
  305. return -1;
  306. }
  307. static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
  308. unsigned long end_pfn)
  309. {
  310. unsigned long old_pgdat_end_pfn =
  311. pgdat->node_start_pfn + pgdat->node_spanned_pages;
  312. if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
  313. pgdat->node_start_pfn = start_pfn;
  314. pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
  315. pgdat->node_start_pfn;
  316. }
  317. static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
  318. {
  319. struct pglist_data *pgdat = zone->zone_pgdat;
  320. int nr_pages = PAGES_PER_SECTION;
  321. int nid = pgdat->node_id;
  322. int zone_type;
  323. unsigned long flags;
  324. int ret;
  325. zone_type = zone - pgdat->node_zones;
  326. ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
  327. if (ret)
  328. return ret;
  329. pgdat_resize_lock(zone->zone_pgdat, &flags);
  330. grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
  331. grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
  332. phys_start_pfn + nr_pages);
  333. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  334. memmap_init_zone(nr_pages, nid, zone_type,
  335. phys_start_pfn, MEMMAP_HOTPLUG);
  336. return 0;
  337. }
  338. static int __meminit __add_section(int nid, struct zone *zone,
  339. unsigned long phys_start_pfn)
  340. {
  341. int nr_pages = PAGES_PER_SECTION;
  342. int ret;
  343. if (pfn_valid(phys_start_pfn))
  344. return -EEXIST;
  345. ret = sparse_add_one_section(zone, phys_start_pfn, nr_pages);
  346. if (ret < 0)
  347. return ret;
  348. ret = __add_zone(zone, phys_start_pfn);
  349. if (ret < 0)
  350. return ret;
  351. return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
  352. }
  353. /*
  354. * Reasonably generic function for adding memory. It is
  355. * expected that archs that support memory hotplug will
  356. * call this function after deciding the zone to which to
  357. * add the new pages.
  358. */
  359. int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
  360. unsigned long nr_pages)
  361. {
  362. unsigned long i;
  363. int err = 0;
  364. int start_sec, end_sec;
  365. /* during initialize mem_map, align hot-added range to section */
  366. start_sec = pfn_to_section_nr(phys_start_pfn);
  367. end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
  368. for (i = start_sec; i <= end_sec; i++) {
  369. err = __add_section(nid, zone, i << PFN_SECTION_SHIFT);
  370. /*
  371. * EEXIST is finally dealt with by ioresource collision
  372. * check. see add_memory() => register_memory_resource()
  373. * Warning will be printed if there is collision.
  374. */
  375. if (err && (err != -EEXIST))
  376. break;
  377. err = 0;
  378. }
  379. return err;
  380. }
  381. EXPORT_SYMBOL_GPL(__add_pages);
  382. #ifdef CONFIG_MEMORY_HOTREMOVE
  383. /* find the smallest valid pfn in the range [start_pfn, end_pfn) */
  384. static int find_smallest_section_pfn(int nid, struct zone *zone,
  385. unsigned long start_pfn,
  386. unsigned long end_pfn)
  387. {
  388. struct mem_section *ms;
  389. for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
  390. ms = __pfn_to_section(start_pfn);
  391. if (unlikely(!valid_section(ms)))
  392. continue;
  393. if (unlikely(pfn_to_nid(start_pfn) != nid))
  394. continue;
  395. if (zone && zone != page_zone(pfn_to_page(start_pfn)))
  396. continue;
  397. return start_pfn;
  398. }
  399. return 0;
  400. }
  401. /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
  402. static int find_biggest_section_pfn(int nid, struct zone *zone,
  403. unsigned long start_pfn,
  404. unsigned long end_pfn)
  405. {
  406. struct mem_section *ms;
  407. unsigned long pfn;
  408. /* pfn is the end pfn of a memory section. */
  409. pfn = end_pfn - 1;
  410. for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
  411. ms = __pfn_to_section(pfn);
  412. if (unlikely(!valid_section(ms)))
  413. continue;
  414. if (unlikely(pfn_to_nid(pfn) != nid))
  415. continue;
  416. if (zone && zone != page_zone(pfn_to_page(pfn)))
  417. continue;
  418. return pfn;
  419. }
  420. return 0;
  421. }
  422. static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
  423. unsigned long end_pfn)
  424. {
  425. unsigned long zone_start_pfn = zone->zone_start_pfn;
  426. unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
  427. unsigned long pfn;
  428. struct mem_section *ms;
  429. int nid = zone_to_nid(zone);
  430. zone_span_writelock(zone);
  431. if (zone_start_pfn == start_pfn) {
  432. /*
  433. * If the section is smallest section in the zone, it need
  434. * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
  435. * In this case, we find second smallest valid mem_section
  436. * for shrinking zone.
  437. */
  438. pfn = find_smallest_section_pfn(nid, zone, end_pfn,
  439. zone_end_pfn);
  440. if (pfn) {
  441. zone->zone_start_pfn = pfn;
  442. zone->spanned_pages = zone_end_pfn - pfn;
  443. }
  444. } else if (zone_end_pfn == end_pfn) {
  445. /*
  446. * If the section is biggest section in the zone, it need
  447. * shrink zone->spanned_pages.
  448. * In this case, we find second biggest valid mem_section for
  449. * shrinking zone.
  450. */
  451. pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
  452. start_pfn);
  453. if (pfn)
  454. zone->spanned_pages = pfn - zone_start_pfn + 1;
  455. }
  456. /*
  457. * The section is not biggest or smallest mem_section in the zone, it
  458. * only creates a hole in the zone. So in this case, we need not
  459. * change the zone. But perhaps, the zone has only hole data. Thus
  460. * it check the zone has only hole or not.
  461. */
  462. pfn = zone_start_pfn;
  463. for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
  464. ms = __pfn_to_section(pfn);
  465. if (unlikely(!valid_section(ms)))
  466. continue;
  467. if (page_zone(pfn_to_page(pfn)) != zone)
  468. continue;
  469. /* If the section is current section, it continues the loop */
  470. if (start_pfn == pfn)
  471. continue;
  472. /* If we find valid section, we have nothing to do */
  473. zone_span_writeunlock(zone);
  474. return;
  475. }
  476. /* The zone has no valid section */
  477. zone->zone_start_pfn = 0;
  478. zone->spanned_pages = 0;
  479. zone_span_writeunlock(zone);
  480. }
  481. static void shrink_pgdat_span(struct pglist_data *pgdat,
  482. unsigned long start_pfn, unsigned long end_pfn)
  483. {
  484. unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
  485. unsigned long pgdat_end_pfn =
  486. pgdat->node_start_pfn + pgdat->node_spanned_pages;
  487. unsigned long pfn;
  488. struct mem_section *ms;
  489. int nid = pgdat->node_id;
  490. if (pgdat_start_pfn == start_pfn) {
  491. /*
  492. * If the section is smallest section in the pgdat, it need
  493. * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
  494. * In this case, we find second smallest valid mem_section
  495. * for shrinking zone.
  496. */
  497. pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
  498. pgdat_end_pfn);
  499. if (pfn) {
  500. pgdat->node_start_pfn = pfn;
  501. pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
  502. }
  503. } else if (pgdat_end_pfn == end_pfn) {
  504. /*
  505. * If the section is biggest section in the pgdat, it need
  506. * shrink pgdat->node_spanned_pages.
  507. * In this case, we find second biggest valid mem_section for
  508. * shrinking zone.
  509. */
  510. pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
  511. start_pfn);
  512. if (pfn)
  513. pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
  514. }
  515. /*
  516. * If the section is not biggest or smallest mem_section in the pgdat,
  517. * it only creates a hole in the pgdat. So in this case, we need not
  518. * change the pgdat.
  519. * But perhaps, the pgdat has only hole data. Thus it check the pgdat
  520. * has only hole or not.
  521. */
  522. pfn = pgdat_start_pfn;
  523. for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
  524. ms = __pfn_to_section(pfn);
  525. if (unlikely(!valid_section(ms)))
  526. continue;
  527. if (pfn_to_nid(pfn) != nid)
  528. continue;
  529. /* If the section is current section, it continues the loop */
  530. if (start_pfn == pfn)
  531. continue;
  532. /* If we find valid section, we have nothing to do */
  533. return;
  534. }
  535. /* The pgdat has no valid section */
  536. pgdat->node_start_pfn = 0;
  537. pgdat->node_spanned_pages = 0;
  538. }
  539. static void __remove_zone(struct zone *zone, unsigned long start_pfn)
  540. {
  541. struct pglist_data *pgdat = zone->zone_pgdat;
  542. int nr_pages = PAGES_PER_SECTION;
  543. int zone_type;
  544. unsigned long flags;
  545. zone_type = zone - pgdat->node_zones;
  546. pgdat_resize_lock(zone->zone_pgdat, &flags);
  547. shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
  548. shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
  549. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  550. }
  551. static int __remove_section(struct zone *zone, struct mem_section *ms)
  552. {
  553. unsigned long start_pfn;
  554. int scn_nr;
  555. int ret = -EINVAL;
  556. if (!valid_section(ms))
  557. return ret;
  558. ret = unregister_memory_section(ms);
  559. if (ret)
  560. return ret;
  561. scn_nr = __section_nr(ms);
  562. start_pfn = section_nr_to_pfn(scn_nr);
  563. __remove_zone(zone, start_pfn);
  564. sparse_remove_one_section(zone, ms);
  565. return 0;
  566. }
  567. /**
  568. * __remove_pages() - remove sections of pages from a zone
  569. * @zone: zone from which pages need to be removed
  570. * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
  571. * @nr_pages: number of pages to remove (must be multiple of section size)
  572. *
  573. * Generic helper function to remove section mappings and sysfs entries
  574. * for the section of the memory we are removing. Caller needs to make
  575. * sure that pages are marked reserved and zones are adjust properly by
  576. * calling offline_pages().
  577. */
  578. int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
  579. unsigned long nr_pages)
  580. {
  581. unsigned long i;
  582. int sections_to_remove;
  583. resource_size_t start, size;
  584. int ret = 0;
  585. /*
  586. * We can only remove entire sections
  587. */
  588. BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
  589. BUG_ON(nr_pages % PAGES_PER_SECTION);
  590. start = phys_start_pfn << PAGE_SHIFT;
  591. size = nr_pages * PAGE_SIZE;
  592. ret = release_mem_region_adjustable(&iomem_resource, start, size);
  593. if (ret) {
  594. resource_size_t endres = start + size - 1;
  595. pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
  596. &start, &endres, ret);
  597. }
  598. sections_to_remove = nr_pages / PAGES_PER_SECTION;
  599. for (i = 0; i < sections_to_remove; i++) {
  600. unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
  601. ret = __remove_section(zone, __pfn_to_section(pfn));
  602. if (ret)
  603. break;
  604. }
  605. return ret;
  606. }
  607. EXPORT_SYMBOL_GPL(__remove_pages);
  608. #endif /* CONFIG_MEMORY_HOTREMOVE */
  609. int set_online_page_callback(online_page_callback_t callback)
  610. {
  611. int rc = -EINVAL;
  612. lock_memory_hotplug();
  613. if (online_page_callback == generic_online_page) {
  614. online_page_callback = callback;
  615. rc = 0;
  616. }
  617. unlock_memory_hotplug();
  618. return rc;
  619. }
  620. EXPORT_SYMBOL_GPL(set_online_page_callback);
  621. int restore_online_page_callback(online_page_callback_t callback)
  622. {
  623. int rc = -EINVAL;
  624. lock_memory_hotplug();
  625. if (online_page_callback == callback) {
  626. online_page_callback = generic_online_page;
  627. rc = 0;
  628. }
  629. unlock_memory_hotplug();
  630. return rc;
  631. }
  632. EXPORT_SYMBOL_GPL(restore_online_page_callback);
  633. void __online_page_set_limits(struct page *page)
  634. {
  635. unsigned long pfn = page_to_pfn(page);
  636. if (pfn >= num_physpages)
  637. num_physpages = pfn + 1;
  638. }
  639. EXPORT_SYMBOL_GPL(__online_page_set_limits);
  640. void __online_page_increment_counters(struct page *page)
  641. {
  642. totalram_pages++;
  643. #ifdef CONFIG_HIGHMEM
  644. if (PageHighMem(page))
  645. totalhigh_pages++;
  646. #endif
  647. }
  648. EXPORT_SYMBOL_GPL(__online_page_increment_counters);
  649. void __online_page_free(struct page *page)
  650. {
  651. ClearPageReserved(page);
  652. init_page_count(page);
  653. __free_page(page);
  654. }
  655. EXPORT_SYMBOL_GPL(__online_page_free);
  656. static void generic_online_page(struct page *page)
  657. {
  658. __online_page_set_limits(page);
  659. __online_page_increment_counters(page);
  660. __online_page_free(page);
  661. }
  662. static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
  663. void *arg)
  664. {
  665. unsigned long i;
  666. unsigned long onlined_pages = *(unsigned long *)arg;
  667. struct page *page;
  668. if (PageReserved(pfn_to_page(start_pfn)))
  669. for (i = 0; i < nr_pages; i++) {
  670. page = pfn_to_page(start_pfn + i);
  671. (*online_page_callback)(page);
  672. onlined_pages++;
  673. }
  674. *(unsigned long *)arg = onlined_pages;
  675. return 0;
  676. }
  677. #ifdef CONFIG_MOVABLE_NODE
  678. /*
  679. * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
  680. * normal memory.
  681. */
  682. static bool can_online_high_movable(struct zone *zone)
  683. {
  684. return true;
  685. }
  686. #else /* CONFIG_MOVABLE_NODE */
  687. /* ensure every online node has NORMAL memory */
  688. static bool can_online_high_movable(struct zone *zone)
  689. {
  690. return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  691. }
  692. #endif /* CONFIG_MOVABLE_NODE */
  693. /* check which state of node_states will be changed when online memory */
  694. static void node_states_check_changes_online(unsigned long nr_pages,
  695. struct zone *zone, struct memory_notify *arg)
  696. {
  697. int nid = zone_to_nid(zone);
  698. enum zone_type zone_last = ZONE_NORMAL;
  699. /*
  700. * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
  701. * contains nodes which have zones of 0...ZONE_NORMAL,
  702. * set zone_last to ZONE_NORMAL.
  703. *
  704. * If we don't have HIGHMEM nor movable node,
  705. * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
  706. * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
  707. */
  708. if (N_MEMORY == N_NORMAL_MEMORY)
  709. zone_last = ZONE_MOVABLE;
  710. /*
  711. * if the memory to be online is in a zone of 0...zone_last, and
  712. * the zones of 0...zone_last don't have memory before online, we will
  713. * need to set the node to node_states[N_NORMAL_MEMORY] after
  714. * the memory is online.
  715. */
  716. if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
  717. arg->status_change_nid_normal = nid;
  718. else
  719. arg->status_change_nid_normal = -1;
  720. #ifdef CONFIG_HIGHMEM
  721. /*
  722. * If we have movable node, node_states[N_HIGH_MEMORY]
  723. * contains nodes which have zones of 0...ZONE_HIGHMEM,
  724. * set zone_last to ZONE_HIGHMEM.
  725. *
  726. * If we don't have movable node, node_states[N_NORMAL_MEMORY]
  727. * contains nodes which have zones of 0...ZONE_MOVABLE,
  728. * set zone_last to ZONE_MOVABLE.
  729. */
  730. zone_last = ZONE_HIGHMEM;
  731. if (N_MEMORY == N_HIGH_MEMORY)
  732. zone_last = ZONE_MOVABLE;
  733. if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
  734. arg->status_change_nid_high = nid;
  735. else
  736. arg->status_change_nid_high = -1;
  737. #else
  738. arg->status_change_nid_high = arg->status_change_nid_normal;
  739. #endif
  740. /*
  741. * if the node don't have memory befor online, we will need to
  742. * set the node to node_states[N_MEMORY] after the memory
  743. * is online.
  744. */
  745. if (!node_state(nid, N_MEMORY))
  746. arg->status_change_nid = nid;
  747. else
  748. arg->status_change_nid = -1;
  749. }
  750. static void node_states_set_node(int node, struct memory_notify *arg)
  751. {
  752. if (arg->status_change_nid_normal >= 0)
  753. node_set_state(node, N_NORMAL_MEMORY);
  754. if (arg->status_change_nid_high >= 0)
  755. node_set_state(node, N_HIGH_MEMORY);
  756. node_set_state(node, N_MEMORY);
  757. }
  758. int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
  759. {
  760. unsigned long flags;
  761. unsigned long onlined_pages = 0;
  762. struct zone *zone;
  763. int need_zonelists_rebuild = 0;
  764. int nid;
  765. int ret;
  766. struct memory_notify arg;
  767. lock_memory_hotplug();
  768. /*
  769. * This doesn't need a lock to do pfn_to_page().
  770. * The section can't be removed here because of the
  771. * memory_block->state_mutex.
  772. */
  773. zone = page_zone(pfn_to_page(pfn));
  774. if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) &&
  775. !can_online_high_movable(zone)) {
  776. unlock_memory_hotplug();
  777. return -1;
  778. }
  779. if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) {
  780. if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) {
  781. unlock_memory_hotplug();
  782. return -1;
  783. }
  784. }
  785. if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) {
  786. if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) {
  787. unlock_memory_hotplug();
  788. return -1;
  789. }
  790. }
  791. /* Previous code may changed the zone of the pfn range */
  792. zone = page_zone(pfn_to_page(pfn));
  793. arg.start_pfn = pfn;
  794. arg.nr_pages = nr_pages;
  795. node_states_check_changes_online(nr_pages, zone, &arg);
  796. nid = page_to_nid(pfn_to_page(pfn));
  797. ret = memory_notify(MEM_GOING_ONLINE, &arg);
  798. ret = notifier_to_errno(ret);
  799. if (ret) {
  800. memory_notify(MEM_CANCEL_ONLINE, &arg);
  801. unlock_memory_hotplug();
  802. return ret;
  803. }
  804. /*
  805. * If this zone is not populated, then it is not in zonelist.
  806. * This means the page allocator ignores this zone.
  807. * So, zonelist must be updated after online.
  808. */
  809. mutex_lock(&zonelists_mutex);
  810. if (!populated_zone(zone)) {
  811. need_zonelists_rebuild = 1;
  812. build_all_zonelists(NULL, zone);
  813. }
  814. ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
  815. online_pages_range);
  816. if (ret) {
  817. if (need_zonelists_rebuild)
  818. zone_pcp_reset(zone);
  819. mutex_unlock(&zonelists_mutex);
  820. printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
  821. (unsigned long long) pfn << PAGE_SHIFT,
  822. (((unsigned long long) pfn + nr_pages)
  823. << PAGE_SHIFT) - 1);
  824. memory_notify(MEM_CANCEL_ONLINE, &arg);
  825. unlock_memory_hotplug();
  826. return ret;
  827. }
  828. zone->managed_pages += onlined_pages;
  829. zone->present_pages += onlined_pages;
  830. pgdat_resize_lock(zone->zone_pgdat, &flags);
  831. zone->zone_pgdat->node_present_pages += onlined_pages;
  832. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  833. if (onlined_pages) {
  834. node_states_set_node(zone_to_nid(zone), &arg);
  835. if (need_zonelists_rebuild)
  836. build_all_zonelists(NULL, NULL);
  837. else
  838. zone_pcp_update(zone);
  839. }
  840. mutex_unlock(&zonelists_mutex);
  841. init_per_zone_wmark_min();
  842. if (onlined_pages)
  843. kswapd_run(zone_to_nid(zone));
  844. vm_total_pages = nr_free_pagecache_pages();
  845. writeback_set_ratelimit();
  846. if (onlined_pages)
  847. memory_notify(MEM_ONLINE, &arg);
  848. unlock_memory_hotplug();
  849. return 0;
  850. }
  851. #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
  852. /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
  853. static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
  854. {
  855. struct pglist_data *pgdat;
  856. unsigned long zones_size[MAX_NR_ZONES] = {0};
  857. unsigned long zholes_size[MAX_NR_ZONES] = {0};
  858. unsigned long start_pfn = start >> PAGE_SHIFT;
  859. pgdat = NODE_DATA(nid);
  860. if (!pgdat) {
  861. pgdat = arch_alloc_nodedata(nid);
  862. if (!pgdat)
  863. return NULL;
  864. arch_refresh_nodedata(nid, pgdat);
  865. }
  866. /* we can use NODE_DATA(nid) from here */
  867. /* init node's zones as empty zones, we don't have any present pages.*/
  868. free_area_init_node(nid, zones_size, start_pfn, zholes_size);
  869. /*
  870. * The node we allocated has no zone fallback lists. For avoiding
  871. * to access not-initialized zonelist, build here.
  872. */
  873. mutex_lock(&zonelists_mutex);
  874. build_all_zonelists(pgdat, NULL);
  875. mutex_unlock(&zonelists_mutex);
  876. return pgdat;
  877. }
  878. static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
  879. {
  880. arch_refresh_nodedata(nid, NULL);
  881. arch_free_nodedata(pgdat);
  882. return;
  883. }
  884. /*
  885. * called by cpu_up() to online a node without onlined memory.
  886. */
  887. int mem_online_node(int nid)
  888. {
  889. pg_data_t *pgdat;
  890. int ret;
  891. lock_memory_hotplug();
  892. pgdat = hotadd_new_pgdat(nid, 0);
  893. if (!pgdat) {
  894. ret = -ENOMEM;
  895. goto out;
  896. }
  897. node_set_online(nid);
  898. ret = register_one_node(nid);
  899. BUG_ON(ret);
  900. out:
  901. unlock_memory_hotplug();
  902. return ret;
  903. }
  904. /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
  905. int __ref add_memory(int nid, u64 start, u64 size)
  906. {
  907. pg_data_t *pgdat = NULL;
  908. bool new_pgdat;
  909. bool new_node;
  910. struct resource *res;
  911. int ret;
  912. lock_memory_hotplug();
  913. res = register_memory_resource(start, size);
  914. ret = -EEXIST;
  915. if (!res)
  916. goto out;
  917. { /* Stupid hack to suppress address-never-null warning */
  918. void *p = NODE_DATA(nid);
  919. new_pgdat = !p;
  920. }
  921. new_node = !node_online(nid);
  922. if (new_node) {
  923. pgdat = hotadd_new_pgdat(nid, start);
  924. ret = -ENOMEM;
  925. if (!pgdat)
  926. goto error;
  927. }
  928. /* call arch's memory hotadd */
  929. ret = arch_add_memory(nid, start, size);
  930. if (ret < 0)
  931. goto error;
  932. /* we online node here. we can't roll back from here. */
  933. node_set_online(nid);
  934. if (new_node) {
  935. ret = register_one_node(nid);
  936. /*
  937. * If sysfs file of new node can't create, cpu on the node
  938. * can't be hot-added. There is no rollback way now.
  939. * So, check by BUG_ON() to catch it reluctantly..
  940. */
  941. BUG_ON(ret);
  942. }
  943. /* create new memmap entry */
  944. firmware_map_add_hotplug(start, start + size, "System RAM");
  945. goto out;
  946. error:
  947. /* rollback pgdat allocation and others */
  948. if (new_pgdat)
  949. rollback_node_hotadd(nid, pgdat);
  950. release_memory_resource(res);
  951. out:
  952. unlock_memory_hotplug();
  953. return ret;
  954. }
  955. EXPORT_SYMBOL_GPL(add_memory);
  956. #ifdef CONFIG_MEMORY_HOTREMOVE
  957. /*
  958. * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
  959. * set and the size of the free page is given by page_order(). Using this,
  960. * the function determines if the pageblock contains only free pages.
  961. * Due to buddy contraints, a free page at least the size of a pageblock will
  962. * be located at the start of the pageblock
  963. */
  964. static inline int pageblock_free(struct page *page)
  965. {
  966. return PageBuddy(page) && page_order(page) >= pageblock_order;
  967. }
  968. /* Return the start of the next active pageblock after a given page */
  969. static struct page *next_active_pageblock(struct page *page)
  970. {
  971. /* Ensure the starting page is pageblock-aligned */
  972. BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));
  973. /* If the entire pageblock is free, move to the end of free page */
  974. if (pageblock_free(page)) {
  975. int order;
  976. /* be careful. we don't have locks, page_order can be changed.*/
  977. order = page_order(page);
  978. if ((order < MAX_ORDER) && (order >= pageblock_order))
  979. return page + (1 << order);
  980. }
  981. return page + pageblock_nr_pages;
  982. }
  983. /* Checks if this range of memory is likely to be hot-removable. */
  984. int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
  985. {
  986. struct page *page = pfn_to_page(start_pfn);
  987. struct page *end_page = page + nr_pages;
  988. /* Check the starting page of each pageblock within the range */
  989. for (; page < end_page; page = next_active_pageblock(page)) {
  990. if (!is_pageblock_removable_nolock(page))
  991. return 0;
  992. cond_resched();
  993. }
  994. /* All pageblocks in the memory block are likely to be hot-removable */
  995. return 1;
  996. }
  997. /*
  998. * Confirm all pages in a range [start, end) is belongs to the same zone.
  999. */
  1000. static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
  1001. {
  1002. unsigned long pfn;
  1003. struct zone *zone = NULL;
  1004. struct page *page;
  1005. int i;
  1006. for (pfn = start_pfn;
  1007. pfn < end_pfn;
  1008. pfn += MAX_ORDER_NR_PAGES) {
  1009. i = 0;
  1010. /* This is just a CONFIG_HOLES_IN_ZONE check.*/
  1011. while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i))
  1012. i++;
  1013. if (i == MAX_ORDER_NR_PAGES)
  1014. continue;
  1015. page = pfn_to_page(pfn + i);
  1016. if (zone && page_zone(page) != zone)
  1017. return 0;
  1018. zone = page_zone(page);
  1019. }
  1020. return 1;
  1021. }
  1022. /*
  1023. * Scanning pfn is much easier than scanning lru list.
  1024. * Scan pfn from start to end and Find LRU page.
  1025. */
  1026. static unsigned long scan_lru_pages(unsigned long start, unsigned long end)
  1027. {
  1028. unsigned long pfn;
  1029. struct page *page;
  1030. for (pfn = start; pfn < end; pfn++) {
  1031. if (pfn_valid(pfn)) {
  1032. page = pfn_to_page(pfn);
  1033. if (PageLRU(page))
  1034. return pfn;
  1035. }
  1036. }
  1037. return 0;
  1038. }
  1039. #define NR_OFFLINE_AT_ONCE_PAGES (256)
  1040. static int
  1041. do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
  1042. {
  1043. unsigned long pfn;
  1044. struct page *page;
  1045. int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
  1046. int not_managed = 0;
  1047. int ret = 0;
  1048. LIST_HEAD(source);
  1049. for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
  1050. if (!pfn_valid(pfn))
  1051. continue;
  1052. page = pfn_to_page(pfn);
  1053. if (!get_page_unless_zero(page))
  1054. continue;
  1055. /*
  1056. * We can skip free pages. And we can only deal with pages on
  1057. * LRU.
  1058. */
  1059. ret = isolate_lru_page(page);
  1060. if (!ret) { /* Success */
  1061. put_page(page);
  1062. list_add_tail(&page->lru, &source);
  1063. move_pages--;
  1064. inc_zone_page_state(page, NR_ISOLATED_ANON +
  1065. page_is_file_cache(page));
  1066. } else {
  1067. #ifdef CONFIG_DEBUG_VM
  1068. printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
  1069. pfn);
  1070. dump_page(page);
  1071. #endif
  1072. put_page(page);
  1073. /* Because we don't have big zone->lock. we should
  1074. check this again here. */
  1075. if (page_count(page)) {
  1076. not_managed++;
  1077. ret = -EBUSY;
  1078. break;
  1079. }
  1080. }
  1081. }
  1082. if (!list_empty(&source)) {
  1083. if (not_managed) {
  1084. putback_lru_pages(&source);
  1085. goto out;
  1086. }
  1087. /*
  1088. * alloc_migrate_target should be improooooved!!
  1089. * migrate_pages returns # of failed pages.
  1090. */
  1091. ret = migrate_pages(&source, alloc_migrate_target, 0,
  1092. MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
  1093. if (ret)
  1094. putback_lru_pages(&source);
  1095. }
  1096. out:
  1097. return ret;
  1098. }
  1099. /*
  1100. * remove from free_area[] and mark all as Reserved.
  1101. */
  1102. static int
  1103. offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
  1104. void *data)
  1105. {
  1106. __offline_isolated_pages(start, start + nr_pages);
  1107. return 0;
  1108. }
  1109. static void
  1110. offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  1111. {
  1112. walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
  1113. offline_isolated_pages_cb);
  1114. }
  1115. /*
  1116. * Check all pages in range, recoreded as memory resource, are isolated.
  1117. */
  1118. static int
  1119. check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
  1120. void *data)
  1121. {
  1122. int ret;
  1123. long offlined = *(long *)data;
  1124. ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
  1125. offlined = nr_pages;
  1126. if (!ret)
  1127. *(long *)data += offlined;
  1128. return ret;
  1129. }
  1130. static long
  1131. check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
  1132. {
  1133. long offlined = 0;
  1134. int ret;
  1135. ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
  1136. check_pages_isolated_cb);
  1137. if (ret < 0)
  1138. offlined = (long)ret;
  1139. return offlined;
  1140. }
  1141. #ifdef CONFIG_MOVABLE_NODE
  1142. /*
  1143. * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
  1144. * normal memory.
  1145. */
  1146. static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
  1147. {
  1148. return true;
  1149. }
  1150. #else /* CONFIG_MOVABLE_NODE */
  1151. /* ensure the node has NORMAL memory if it is still online */
  1152. static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
  1153. {
  1154. struct pglist_data *pgdat = zone->zone_pgdat;
  1155. unsigned long present_pages = 0;
  1156. enum zone_type zt;
  1157. for (zt = 0; zt <= ZONE_NORMAL; zt++)
  1158. present_pages += pgdat->node_zones[zt].present_pages;
  1159. if (present_pages > nr_pages)
  1160. return true;
  1161. present_pages = 0;
  1162. for (; zt <= ZONE_MOVABLE; zt++)
  1163. present_pages += pgdat->node_zones[zt].present_pages;
  1164. /*
  1165. * we can't offline the last normal memory until all
  1166. * higher memory is offlined.
  1167. */
  1168. return present_pages == 0;
  1169. }
  1170. #endif /* CONFIG_MOVABLE_NODE */
  1171. /* check which state of node_states will be changed when offline memory */
  1172. static void node_states_check_changes_offline(unsigned long nr_pages,
  1173. struct zone *zone, struct memory_notify *arg)
  1174. {
  1175. struct pglist_data *pgdat = zone->zone_pgdat;
  1176. unsigned long present_pages = 0;
  1177. enum zone_type zt, zone_last = ZONE_NORMAL;
  1178. /*
  1179. * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
  1180. * contains nodes which have zones of 0...ZONE_NORMAL,
  1181. * set zone_last to ZONE_NORMAL.
  1182. *
  1183. * If we don't have HIGHMEM nor movable node,
  1184. * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
  1185. * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
  1186. */
  1187. if (N_MEMORY == N_NORMAL_MEMORY)
  1188. zone_last = ZONE_MOVABLE;
  1189. /*
  1190. * check whether node_states[N_NORMAL_MEMORY] will be changed.
  1191. * If the memory to be offline is in a zone of 0...zone_last,
  1192. * and it is the last present memory, 0...zone_last will
  1193. * become empty after offline , thus we can determind we will
  1194. * need to clear the node from node_states[N_NORMAL_MEMORY].
  1195. */
  1196. for (zt = 0; zt <= zone_last; zt++)
  1197. present_pages += pgdat->node_zones[zt].present_pages;
  1198. if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
  1199. arg->status_change_nid_normal = zone_to_nid(zone);
  1200. else
  1201. arg->status_change_nid_normal = -1;
  1202. #ifdef CONFIG_HIGHMEM
  1203. /*
  1204. * If we have movable node, node_states[N_HIGH_MEMORY]
  1205. * contains nodes which have zones of 0...ZONE_HIGHMEM,
  1206. * set zone_last to ZONE_HIGHMEM.
  1207. *
  1208. * If we don't have movable node, node_states[N_NORMAL_MEMORY]
  1209. * contains nodes which have zones of 0...ZONE_MOVABLE,
  1210. * set zone_last to ZONE_MOVABLE.
  1211. */
  1212. zone_last = ZONE_HIGHMEM;
  1213. if (N_MEMORY == N_HIGH_MEMORY)
  1214. zone_last = ZONE_MOVABLE;
  1215. for (; zt <= zone_last; zt++)
  1216. present_pages += pgdat->node_zones[zt].present_pages;
  1217. if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
  1218. arg->status_change_nid_high = zone_to_nid(zone);
  1219. else
  1220. arg->status_change_nid_high = -1;
  1221. #else
  1222. arg->status_change_nid_high = arg->status_change_nid_normal;
  1223. #endif
  1224. /*
  1225. * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
  1226. */
  1227. zone_last = ZONE_MOVABLE;
  1228. /*
  1229. * check whether node_states[N_HIGH_MEMORY] will be changed
  1230. * If we try to offline the last present @nr_pages from the node,
  1231. * we can determind we will need to clear the node from
  1232. * node_states[N_HIGH_MEMORY].
  1233. */
  1234. for (; zt <= zone_last; zt++)
  1235. present_pages += pgdat->node_zones[zt].present_pages;
  1236. if (nr_pages >= present_pages)
  1237. arg->status_change_nid = zone_to_nid(zone);
  1238. else
  1239. arg->status_change_nid = -1;
  1240. }
  1241. static void node_states_clear_node(int node, struct memory_notify *arg)
  1242. {
  1243. if (arg->status_change_nid_normal >= 0)
  1244. node_clear_state(node, N_NORMAL_MEMORY);
  1245. if ((N_MEMORY != N_NORMAL_MEMORY) &&
  1246. (arg->status_change_nid_high >= 0))
  1247. node_clear_state(node, N_HIGH_MEMORY);
  1248. if ((N_MEMORY != N_HIGH_MEMORY) &&
  1249. (arg->status_change_nid >= 0))
  1250. node_clear_state(node, N_MEMORY);
  1251. }
  1252. static int __ref __offline_pages(unsigned long start_pfn,
  1253. unsigned long end_pfn, unsigned long timeout)
  1254. {
  1255. unsigned long pfn, nr_pages, expire;
  1256. long offlined_pages;
  1257. int ret, drain, retry_max, node;
  1258. unsigned long flags;
  1259. struct zone *zone;
  1260. struct memory_notify arg;
  1261. BUG_ON(start_pfn >= end_pfn);
  1262. /* at least, alignment against pageblock is necessary */
  1263. if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
  1264. return -EINVAL;
  1265. if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
  1266. return -EINVAL;
  1267. /* This makes hotplug much easier...and readable.
  1268. we assume this for now. .*/
  1269. if (!test_pages_in_a_zone(start_pfn, end_pfn))
  1270. return -EINVAL;
  1271. lock_memory_hotplug();
  1272. zone = page_zone(pfn_to_page(start_pfn));
  1273. node = zone_to_nid(zone);
  1274. nr_pages = end_pfn - start_pfn;
  1275. ret = -EINVAL;
  1276. if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
  1277. goto out;
  1278. /* set above range as isolated */
  1279. ret = start_isolate_page_range(start_pfn, end_pfn,
  1280. MIGRATE_MOVABLE, true);
  1281. if (ret)
  1282. goto out;
  1283. arg.start_pfn = start_pfn;
  1284. arg.nr_pages = nr_pages;
  1285. node_states_check_changes_offline(nr_pages, zone, &arg);
  1286. ret = memory_notify(MEM_GOING_OFFLINE, &arg);
  1287. ret = notifier_to_errno(ret);
  1288. if (ret)
  1289. goto failed_removal;
  1290. pfn = start_pfn;
  1291. expire = jiffies + timeout;
  1292. drain = 0;
  1293. retry_max = 5;
  1294. repeat:
  1295. /* start memory hot removal */
  1296. ret = -EAGAIN;
  1297. if (time_after(jiffies, expire))
  1298. goto failed_removal;
  1299. ret = -EINTR;
  1300. if (signal_pending(current))
  1301. goto failed_removal;
  1302. ret = 0;
  1303. if (drain) {
  1304. lru_add_drain_all();
  1305. cond_resched();
  1306. drain_all_pages();
  1307. }
  1308. pfn = scan_lru_pages(start_pfn, end_pfn);
  1309. if (pfn) { /* We have page on LRU */
  1310. ret = do_migrate_range(pfn, end_pfn);
  1311. if (!ret) {
  1312. drain = 1;
  1313. goto repeat;
  1314. } else {
  1315. if (ret < 0)
  1316. if (--retry_max == 0)
  1317. goto failed_removal;
  1318. yield();
  1319. drain = 1;
  1320. goto repeat;
  1321. }
  1322. }
  1323. /* drain all zone's lru pagevec, this is asynchronous... */
  1324. lru_add_drain_all();
  1325. yield();
  1326. /* drain pcp pages, this is synchronous. */
  1327. drain_all_pages();
  1328. /* check again */
  1329. offlined_pages = check_pages_isolated(start_pfn, end_pfn);
  1330. if (offlined_pages < 0) {
  1331. ret = -EBUSY;
  1332. goto failed_removal;
  1333. }
  1334. printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
  1335. /* Ok, all of our target is isolated.
  1336. We cannot do rollback at this point. */
  1337. offline_isolated_pages(start_pfn, end_pfn);
  1338. /* reset pagetype flags and makes migrate type to be MOVABLE */
  1339. undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  1340. /* removal success */
  1341. zone->managed_pages -= offlined_pages;
  1342. zone->present_pages -= offlined_pages;
  1343. pgdat_resize_lock(zone->zone_pgdat, &flags);
  1344. zone->zone_pgdat->node_present_pages -= offlined_pages;
  1345. pgdat_resize_unlock(zone->zone_pgdat, &flags);
  1346. totalram_pages -= offlined_pages;
  1347. init_per_zone_wmark_min();
  1348. if (!populated_zone(zone)) {
  1349. zone_pcp_reset(zone);
  1350. mutex_lock(&zonelists_mutex);
  1351. build_all_zonelists(NULL, NULL);
  1352. mutex_unlock(&zonelists_mutex);
  1353. } else
  1354. zone_pcp_update(zone);
  1355. node_states_clear_node(node, &arg);
  1356. if (arg.status_change_nid >= 0)
  1357. kswapd_stop(node);
  1358. vm_total_pages = nr_free_pagecache_pages();
  1359. writeback_set_ratelimit();
  1360. memory_notify(MEM_OFFLINE, &arg);
  1361. unlock_memory_hotplug();
  1362. return 0;
  1363. failed_removal:
  1364. printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
  1365. (unsigned long long) start_pfn << PAGE_SHIFT,
  1366. ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
  1367. memory_notify(MEM_CANCEL_OFFLINE, &arg);
  1368. /* pushback to free area */
  1369. undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  1370. out:
  1371. unlock_memory_hotplug();
  1372. return ret;
  1373. }
  1374. int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
  1375. {
  1376. return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
  1377. }
  1378. /**
  1379. * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
  1380. * @start_pfn: start pfn of the memory range
  1381. * @end_pfn: end pfn of the memory range
  1382. * @arg: argument passed to func
  1383. * @func: callback for each memory section walked
  1384. *
  1385. * This function walks through all present mem sections in range
  1386. * [start_pfn, end_pfn) and call func on each mem section.
  1387. *
  1388. * Returns the return value of func.
  1389. */
  1390. static int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
  1391. void *arg, int (*func)(struct memory_block *, void *))
  1392. {
  1393. struct memory_block *mem = NULL;
  1394. struct mem_section *section;
  1395. unsigned long pfn, section_nr;
  1396. int ret;
  1397. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  1398. section_nr = pfn_to_section_nr(pfn);
  1399. if (!present_section_nr(section_nr))
  1400. continue;
  1401. section = __nr_to_section(section_nr);
  1402. /* same memblock? */
  1403. if (mem)
  1404. if ((section_nr >= mem->start_section_nr) &&
  1405. (section_nr <= mem->end_section_nr))
  1406. continue;
  1407. mem = find_memory_block_hinted(section, mem);
  1408. if (!mem)
  1409. continue;
  1410. ret = func(mem, arg);
  1411. if (ret) {
  1412. kobject_put(&mem->dev.kobj);
  1413. return ret;
  1414. }
  1415. }
  1416. if (mem)
  1417. kobject_put(&mem->dev.kobj);
  1418. return 0;
  1419. }
  1420. /**
  1421. * offline_memory_block_cb - callback function for offlining memory block
  1422. * @mem: the memory block to be offlined
  1423. * @arg: buffer to hold error msg
  1424. *
  1425. * Always return 0, and put the error msg in arg if any.
  1426. */
  1427. static int offline_memory_block_cb(struct memory_block *mem, void *arg)
  1428. {
  1429. int *ret = arg;
  1430. int error = offline_memory_block(mem);
  1431. if (error != 0 && *ret == 0)
  1432. *ret = error;
  1433. return 0;
  1434. }
  1435. static int is_memblock_offlined_cb(struct memory_block *mem, void *arg)
  1436. {
  1437. int ret = !is_memblock_offlined(mem);
  1438. if (unlikely(ret)) {
  1439. phys_addr_t beginpa, endpa;
  1440. beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
  1441. endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
  1442. pr_warn("removing memory fails, because memory "
  1443. "[%pa-%pa] is onlined\n",
  1444. &beginpa, &endpa);
  1445. }
  1446. return ret;
  1447. }
  1448. static int check_cpu_on_node(void *data)
  1449. {
  1450. struct pglist_data *pgdat = data;
  1451. int cpu;
  1452. for_each_present_cpu(cpu) {
  1453. if (cpu_to_node(cpu) == pgdat->node_id)
  1454. /*
  1455. * the cpu on this node isn't removed, and we can't
  1456. * offline this node.
  1457. */
  1458. return -EBUSY;
  1459. }
  1460. return 0;
  1461. }
  1462. static void unmap_cpu_on_node(void *data)
  1463. {
  1464. #ifdef CONFIG_ACPI_NUMA
  1465. struct pglist_data *pgdat = data;
  1466. int cpu;
  1467. for_each_possible_cpu(cpu)
  1468. if (cpu_to_node(cpu) == pgdat->node_id)
  1469. numa_clear_node(cpu);
  1470. #endif
  1471. }
  1472. static int check_and_unmap_cpu_on_node(void *data)
  1473. {
  1474. int ret = check_cpu_on_node(data);
  1475. if (ret)
  1476. return ret;
  1477. /*
  1478. * the node will be offlined when we come here, so we can clear
  1479. * the cpu_to_node() now.
  1480. */
  1481. unmap_cpu_on_node(data);
  1482. return 0;
  1483. }
  1484. /* offline the node if all memory sections of this node are removed */
  1485. void try_offline_node(int nid)
  1486. {
  1487. pg_data_t *pgdat = NODE_DATA(nid);
  1488. unsigned long start_pfn = pgdat->node_start_pfn;
  1489. unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
  1490. unsigned long pfn;
  1491. struct page *pgdat_page = virt_to_page(pgdat);
  1492. int i;
  1493. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  1494. unsigned long section_nr = pfn_to_section_nr(pfn);
  1495. if (!present_section_nr(section_nr))
  1496. continue;
  1497. if (pfn_to_nid(pfn) != nid)
  1498. continue;
  1499. /*
  1500. * some memory sections of this node are not removed, and we
  1501. * can't offline node now.
  1502. */
  1503. return;
  1504. }
  1505. if (stop_machine(check_and_unmap_cpu_on_node, pgdat, NULL))
  1506. return;
  1507. /*
  1508. * all memory/cpu of this node are removed, we can offline this
  1509. * node now.
  1510. */
  1511. node_set_offline(nid);
  1512. unregister_one_node(nid);
  1513. if (!PageSlab(pgdat_page) && !PageCompound(pgdat_page))
  1514. /* node data is allocated from boot memory */
  1515. return;
  1516. /* free waittable in each zone */
  1517. for (i = 0; i < MAX_NR_ZONES; i++) {
  1518. struct zone *zone = pgdat->node_zones + i;
  1519. /*
  1520. * wait_table may be allocated from boot memory,
  1521. * here only free if it's allocated by vmalloc.
  1522. */
  1523. if (is_vmalloc_addr(zone->wait_table))
  1524. vfree(zone->wait_table);
  1525. }
  1526. /*
  1527. * Since there is no way to guarentee the address of pgdat/zone is not
  1528. * on stack of any kernel threads or used by other kernel objects
  1529. * without reference counting or other symchronizing method, do not
  1530. * reset node_data and free pgdat here. Just reset it to 0 and reuse
  1531. * the memory when the node is online again.
  1532. */
  1533. memset(pgdat, 0, sizeof(*pgdat));
  1534. }
  1535. EXPORT_SYMBOL(try_offline_node);
  1536. int __ref remove_memory(int nid, u64 start, u64 size)
  1537. {
  1538. unsigned long start_pfn, end_pfn;
  1539. int ret = 0;
  1540. int retry = 1;
  1541. start_pfn = PFN_DOWN(start);
  1542. end_pfn = PFN_UP(start + size - 1);
  1543. /*
  1544. * When CONFIG_MEMCG is on, one memory block may be used by other
  1545. * blocks to store page cgroup when onlining pages. But we don't know
  1546. * in what order pages are onlined. So we iterate twice to offline
  1547. * memory:
  1548. * 1st iterate: offline every non primary memory block.
  1549. * 2nd iterate: offline primary (i.e. first added) memory block.
  1550. */
  1551. repeat:
  1552. walk_memory_range(start_pfn, end_pfn, &ret,
  1553. offline_memory_block_cb);
  1554. if (ret) {
  1555. if (!retry)
  1556. return ret;
  1557. retry = 0;
  1558. ret = 0;
  1559. goto repeat;
  1560. }
  1561. lock_memory_hotplug();
  1562. /*
  1563. * we have offlined all memory blocks like this:
  1564. * 1. lock memory hotplug
  1565. * 2. offline a memory block
  1566. * 3. unlock memory hotplug
  1567. *
  1568. * repeat step1-3 to offline the memory block. All memory blocks
  1569. * must be offlined before removing memory. But we don't hold the
  1570. * lock in the whole operation. So we should check whether all
  1571. * memory blocks are offlined.
  1572. */
  1573. ret = walk_memory_range(start_pfn, end_pfn, NULL,
  1574. is_memblock_offlined_cb);
  1575. if (ret) {
  1576. unlock_memory_hotplug();
  1577. return ret;
  1578. }
  1579. /* remove memmap entry */
  1580. firmware_map_remove(start, start + size, "System RAM");
  1581. arch_remove_memory(start, size);
  1582. try_offline_node(nid);
  1583. unlock_memory_hotplug();
  1584. return 0;
  1585. }
  1586. #else
  1587. int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
  1588. {
  1589. return -EINVAL;
  1590. }
  1591. int remove_memory(int nid, u64 start, u64 size)
  1592. {
  1593. return -EINVAL;
  1594. }
  1595. #endif /* CONFIG_MEMORY_HOTREMOVE */
  1596. EXPORT_SYMBOL_GPL(remove_memory);