bootmem.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847
  1. /*
  2. * bootmem - A boot-time physical memory allocator and configurator
  3. *
  4. * Copyright (C) 1999 Ingo Molnar
  5. * 1999 Kanoj Sarcar, SGI
  6. * 2008 Johannes Weiner
  7. *
  8. * Access to this subsystem has to be serialized externally (which is true
  9. * for the boot process anyway).
  10. */
  11. #include <linux/init.h>
  12. #include <linux/pfn.h>
  13. #include <linux/slab.h>
  14. #include <linux/bootmem.h>
  15. #include <linux/export.h>
  16. #include <linux/kmemleak.h>
  17. #include <linux/range.h>
  18. #include <linux/memblock.h>
  19. #include <asm/bug.h>
  20. #include <asm/io.h>
  21. #include <asm/processor.h>
  22. #include "internal.h"
  23. #ifndef CONFIG_NEED_MULTIPLE_NODES
  24. struct pglist_data __refdata contig_page_data = {
  25. .bdata = &bootmem_node_data[0]
  26. };
  27. EXPORT_SYMBOL(contig_page_data);
  28. #endif
  29. unsigned long max_low_pfn;
  30. unsigned long min_low_pfn;
  31. unsigned long max_pfn;
  32. bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
  33. static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
  34. static int bootmem_debug;
  35. static int __init bootmem_debug_setup(char *buf)
  36. {
  37. bootmem_debug = 1;
  38. return 0;
  39. }
  40. early_param("bootmem_debug", bootmem_debug_setup);
  41. #define bdebug(fmt, args...) ({ \
  42. if (unlikely(bootmem_debug)) \
  43. printk(KERN_INFO \
  44. "bootmem::%s " fmt, \
  45. __func__, ## args); \
  46. })
  47. static unsigned long __init bootmap_bytes(unsigned long pages)
  48. {
  49. unsigned long bytes = DIV_ROUND_UP(pages, 8);
  50. return ALIGN(bytes, sizeof(long));
  51. }
  52. /**
  53. * bootmem_bootmap_pages - calculate bitmap size in pages
  54. * @pages: number of pages the bitmap has to represent
  55. */
  56. unsigned long __init bootmem_bootmap_pages(unsigned long pages)
  57. {
  58. unsigned long bytes = bootmap_bytes(pages);
  59. return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
  60. }
  61. /*
  62. * link bdata in order
  63. */
  64. static void __init link_bootmem(bootmem_data_t *bdata)
  65. {
  66. bootmem_data_t *ent;
  67. list_for_each_entry(ent, &bdata_list, list) {
  68. if (bdata->node_min_pfn < ent->node_min_pfn) {
  69. list_add_tail(&bdata->list, &ent->list);
  70. return;
  71. }
  72. }
  73. list_add_tail(&bdata->list, &bdata_list);
  74. }
  75. /*
  76. * Called once to set up the allocator itself.
  77. */
  78. static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
  79. unsigned long mapstart, unsigned long start, unsigned long end)
  80. {
  81. unsigned long mapsize;
  82. mminit_validate_memmodel_limits(&start, &end);
  83. bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
  84. bdata->node_min_pfn = start;
  85. bdata->node_low_pfn = end;
  86. link_bootmem(bdata);
  87. /*
  88. * Initially all pages are reserved - setup_arch() has to
  89. * register free RAM areas explicitly.
  90. */
  91. mapsize = bootmap_bytes(end - start);
  92. memset(bdata->node_bootmem_map, 0xff, mapsize);
  93. bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
  94. bdata - bootmem_node_data, start, mapstart, end, mapsize);
  95. return mapsize;
  96. }
  97. /**
  98. * init_bootmem_node - register a node as boot memory
  99. * @pgdat: node to register
  100. * @freepfn: pfn where the bitmap for this node is to be placed
  101. * @startpfn: first pfn on the node
  102. * @endpfn: first pfn after the node
  103. *
  104. * Returns the number of bytes needed to hold the bitmap for this node.
  105. */
  106. unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
  107. unsigned long startpfn, unsigned long endpfn)
  108. {
  109. return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
  110. }
  111. /**
  112. * init_bootmem - register boot memory
  113. * @start: pfn where the bitmap is to be placed
  114. * @pages: number of available physical pages
  115. *
  116. * Returns the number of bytes needed to hold the bitmap.
  117. */
  118. unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
  119. {
  120. max_low_pfn = pages;
  121. min_low_pfn = start;
  122. return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
  123. }
  124. /*
  125. * free_bootmem_late - free bootmem pages directly to page allocator
  126. * @addr: starting physical address of the range
  127. * @size: size of the range in bytes
  128. *
  129. * This is only useful when the bootmem allocator has already been torn
  130. * down, but we are still initializing the system. Pages are given directly
  131. * to the page allocator, no bootmem metadata is updated because it is gone.
  132. */
  133. void __init free_bootmem_late(unsigned long physaddr, unsigned long size)
  134. {
  135. unsigned long cursor, end;
  136. kmemleak_free_part(__va(physaddr), size);
  137. cursor = PFN_UP(physaddr);
  138. end = PFN_DOWN(physaddr + size);
  139. for (; cursor < end; cursor++) {
  140. __free_pages_bootmem(pfn_to_page(cursor), 0);
  141. totalram_pages++;
  142. }
  143. }
  144. static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
  145. {
  146. struct page *page;
  147. unsigned long start, end, pages, count = 0;
  148. if (!bdata->node_bootmem_map)
  149. return 0;
  150. start = bdata->node_min_pfn;
  151. end = bdata->node_low_pfn;
  152. bdebug("nid=%td start=%lx end=%lx\n",
  153. bdata - bootmem_node_data, start, end);
  154. while (start < end) {
  155. unsigned long *map, idx, vec;
  156. map = bdata->node_bootmem_map;
  157. idx = start - bdata->node_min_pfn;
  158. vec = ~map[idx / BITS_PER_LONG];
  159. /*
  160. * If we have a properly aligned and fully unreserved
  161. * BITS_PER_LONG block of pages in front of us, free
  162. * it in one go.
  163. */
  164. if (IS_ALIGNED(start, BITS_PER_LONG) && vec == ~0UL) {
  165. int order = ilog2(BITS_PER_LONG);
  166. __free_pages_bootmem(pfn_to_page(start), order);
  167. count += BITS_PER_LONG;
  168. start += BITS_PER_LONG;
  169. } else {
  170. unsigned long off = 0;
  171. vec >>= start & (BITS_PER_LONG - 1);
  172. while (vec) {
  173. if (vec & 1) {
  174. page = pfn_to_page(start + off);
  175. __free_pages_bootmem(page, 0);
  176. count++;
  177. }
  178. vec >>= 1;
  179. off++;
  180. }
  181. start = ALIGN(start + 1, BITS_PER_LONG);
  182. }
  183. }
  184. page = virt_to_page(bdata->node_bootmem_map);
  185. pages = bdata->node_low_pfn - bdata->node_min_pfn;
  186. pages = bootmem_bootmap_pages(pages);
  187. count += pages;
  188. while (pages--)
  189. __free_pages_bootmem(page++, 0);
  190. bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
  191. return count;
  192. }
  193. static void reset_node_lowmem_managed_pages(pg_data_t *pgdat)
  194. {
  195. struct zone *z;
  196. /*
  197. * In free_area_init_core(), highmem zone's managed_pages is set to
  198. * present_pages, and bootmem allocator doesn't allocate from highmem
  199. * zones. So there's no need to recalculate managed_pages because all
  200. * highmem pages will be managed by the buddy system. Here highmem
  201. * zone also includes highmem movable zone.
  202. */
  203. for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
  204. if (!is_highmem(z))
  205. z->managed_pages = 0;
  206. }
  207. /**
  208. * free_all_bootmem_node - release a node's free pages to the buddy allocator
  209. * @pgdat: node to be released
  210. *
  211. * Returns the number of pages actually released.
  212. */
  213. unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
  214. {
  215. register_page_bootmem_info_node(pgdat);
  216. reset_node_lowmem_managed_pages(pgdat);
  217. return free_all_bootmem_core(pgdat->bdata);
  218. }
  219. /**
  220. * free_all_bootmem - release free pages to the buddy allocator
  221. *
  222. * Returns the number of pages actually released.
  223. */
  224. unsigned long __init free_all_bootmem(void)
  225. {
  226. unsigned long total_pages = 0;
  227. bootmem_data_t *bdata;
  228. struct pglist_data *pgdat;
  229. for_each_online_pgdat(pgdat)
  230. reset_node_lowmem_managed_pages(pgdat);
  231. list_for_each_entry(bdata, &bdata_list, list)
  232. total_pages += free_all_bootmem_core(bdata);
  233. return total_pages;
  234. }
  235. static void __init __free(bootmem_data_t *bdata,
  236. unsigned long sidx, unsigned long eidx)
  237. {
  238. unsigned long idx;
  239. bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
  240. sidx + bdata->node_min_pfn,
  241. eidx + bdata->node_min_pfn);
  242. if (bdata->hint_idx > sidx)
  243. bdata->hint_idx = sidx;
  244. for (idx = sidx; idx < eidx; idx++)
  245. if (!test_and_clear_bit(idx, bdata->node_bootmem_map))
  246. BUG();
  247. }
  248. static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,
  249. unsigned long eidx, int flags)
  250. {
  251. unsigned long idx;
  252. int exclusive = flags & BOOTMEM_EXCLUSIVE;
  253. bdebug("nid=%td start=%lx end=%lx flags=%x\n",
  254. bdata - bootmem_node_data,
  255. sidx + bdata->node_min_pfn,
  256. eidx + bdata->node_min_pfn,
  257. flags);
  258. for (idx = sidx; idx < eidx; idx++)
  259. if (test_and_set_bit(idx, bdata->node_bootmem_map)) {
  260. if (exclusive) {
  261. __free(bdata, sidx, idx);
  262. return -EBUSY;
  263. }
  264. bdebug("silent double reserve of PFN %lx\n",
  265. idx + bdata->node_min_pfn);
  266. }
  267. return 0;
  268. }
  269. static int __init mark_bootmem_node(bootmem_data_t *bdata,
  270. unsigned long start, unsigned long end,
  271. int reserve, int flags)
  272. {
  273. unsigned long sidx, eidx;
  274. bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n",
  275. bdata - bootmem_node_data, start, end, reserve, flags);
  276. BUG_ON(start < bdata->node_min_pfn);
  277. BUG_ON(end > bdata->node_low_pfn);
  278. sidx = start - bdata->node_min_pfn;
  279. eidx = end - bdata->node_min_pfn;
  280. if (reserve)
  281. return __reserve(bdata, sidx, eidx, flags);
  282. else
  283. __free(bdata, sidx, eidx);
  284. return 0;
  285. }
  286. static int __init mark_bootmem(unsigned long start, unsigned long end,
  287. int reserve, int flags)
  288. {
  289. unsigned long pos;
  290. bootmem_data_t *bdata;
  291. pos = start;
  292. list_for_each_entry(bdata, &bdata_list, list) {
  293. int err;
  294. unsigned long max;
  295. if (pos < bdata->node_min_pfn ||
  296. pos >= bdata->node_low_pfn) {
  297. BUG_ON(pos != start);
  298. continue;
  299. }
  300. max = min(bdata->node_low_pfn, end);
  301. err = mark_bootmem_node(bdata, pos, max, reserve, flags);
  302. if (reserve && err) {
  303. mark_bootmem(start, pos, 0, 0);
  304. return err;
  305. }
  306. if (max == end)
  307. return 0;
  308. pos = bdata->node_low_pfn;
  309. }
  310. BUG();
  311. }
  312. /**
  313. * free_bootmem_node - mark a page range as usable
  314. * @pgdat: node the range resides on
  315. * @physaddr: starting address of the range
  316. * @size: size of the range in bytes
  317. *
  318. * Partial pages will be considered reserved and left as they are.
  319. *
  320. * The range must reside completely on the specified node.
  321. */
  322. void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
  323. unsigned long size)
  324. {
  325. unsigned long start, end;
  326. kmemleak_free_part(__va(physaddr), size);
  327. start = PFN_UP(physaddr);
  328. end = PFN_DOWN(physaddr + size);
  329. mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
  330. }
  331. /**
  332. * free_bootmem - mark a page range as usable
  333. * @addr: starting physical address of the range
  334. * @size: size of the range in bytes
  335. *
  336. * Partial pages will be considered reserved and left as they are.
  337. *
  338. * The range must be contiguous but may span node boundaries.
  339. */
  340. void __init free_bootmem(unsigned long physaddr, unsigned long size)
  341. {
  342. unsigned long start, end;
  343. kmemleak_free_part(__va(physaddr), size);
  344. start = PFN_UP(physaddr);
  345. end = PFN_DOWN(physaddr + size);
  346. mark_bootmem(start, end, 0, 0);
  347. }
  348. /**
  349. * reserve_bootmem_node - mark a page range as reserved
  350. * @pgdat: node the range resides on
  351. * @physaddr: starting address of the range
  352. * @size: size of the range in bytes
  353. * @flags: reservation flags (see linux/bootmem.h)
  354. *
  355. * Partial pages will be reserved.
  356. *
  357. * The range must reside completely on the specified node.
  358. */
  359. int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
  360. unsigned long size, int flags)
  361. {
  362. unsigned long start, end;
  363. start = PFN_DOWN(physaddr);
  364. end = PFN_UP(physaddr + size);
  365. return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
  366. }
  367. /**
  368. * reserve_bootmem - mark a page range as reserved
  369. * @addr: starting address of the range
  370. * @size: size of the range in bytes
  371. * @flags: reservation flags (see linux/bootmem.h)
  372. *
  373. * Partial pages will be reserved.
  374. *
  375. * The range must be contiguous but may span node boundaries.
  376. */
  377. int __init reserve_bootmem(unsigned long addr, unsigned long size,
  378. int flags)
  379. {
  380. unsigned long start, end;
  381. start = PFN_DOWN(addr);
  382. end = PFN_UP(addr + size);
  383. return mark_bootmem(start, end, 1, flags);
  384. }
  385. static unsigned long __init align_idx(struct bootmem_data *bdata,
  386. unsigned long idx, unsigned long step)
  387. {
  388. unsigned long base = bdata->node_min_pfn;
  389. /*
  390. * Align the index with respect to the node start so that the
  391. * combination of both satisfies the requested alignment.
  392. */
  393. return ALIGN(base + idx, step) - base;
  394. }
  395. static unsigned long __init align_off(struct bootmem_data *bdata,
  396. unsigned long off, unsigned long align)
  397. {
  398. unsigned long base = PFN_PHYS(bdata->node_min_pfn);
  399. /* Same as align_idx for byte offsets */
  400. return ALIGN(base + off, align) - base;
  401. }
  402. static void * __init alloc_bootmem_bdata(struct bootmem_data *bdata,
  403. unsigned long size, unsigned long align,
  404. unsigned long goal, unsigned long limit)
  405. {
  406. unsigned long fallback = 0;
  407. unsigned long min, max, start, sidx, midx, step;
  408. bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
  409. bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
  410. align, goal, limit);
  411. BUG_ON(!size);
  412. BUG_ON(align & (align - 1));
  413. BUG_ON(limit && goal + size > limit);
  414. if (!bdata->node_bootmem_map)
  415. return NULL;
  416. min = bdata->node_min_pfn;
  417. max = bdata->node_low_pfn;
  418. goal >>= PAGE_SHIFT;
  419. limit >>= PAGE_SHIFT;
  420. if (limit && max > limit)
  421. max = limit;
  422. if (max <= min)
  423. return NULL;
  424. step = max(align >> PAGE_SHIFT, 1UL);
  425. if (goal && min < goal && goal < max)
  426. start = ALIGN(goal, step);
  427. else
  428. start = ALIGN(min, step);
  429. sidx = start - bdata->node_min_pfn;
  430. midx = max - bdata->node_min_pfn;
  431. if (bdata->hint_idx > sidx) {
  432. /*
  433. * Handle the valid case of sidx being zero and still
  434. * catch the fallback below.
  435. */
  436. fallback = sidx + 1;
  437. sidx = align_idx(bdata, bdata->hint_idx, step);
  438. }
  439. while (1) {
  440. int merge;
  441. void *region;
  442. unsigned long eidx, i, start_off, end_off;
  443. find_block:
  444. sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
  445. sidx = align_idx(bdata, sidx, step);
  446. eidx = sidx + PFN_UP(size);
  447. if (sidx >= midx || eidx > midx)
  448. break;
  449. for (i = sidx; i < eidx; i++)
  450. if (test_bit(i, bdata->node_bootmem_map)) {
  451. sidx = align_idx(bdata, i, step);
  452. if (sidx == i)
  453. sidx += step;
  454. goto find_block;
  455. }
  456. if (bdata->last_end_off & (PAGE_SIZE - 1) &&
  457. PFN_DOWN(bdata->last_end_off) + 1 == sidx)
  458. start_off = align_off(bdata, bdata->last_end_off, align);
  459. else
  460. start_off = PFN_PHYS(sidx);
  461. merge = PFN_DOWN(start_off) < sidx;
  462. end_off = start_off + size;
  463. bdata->last_end_off = end_off;
  464. bdata->hint_idx = PFN_UP(end_off);
  465. /*
  466. * Reserve the area now:
  467. */
  468. if (__reserve(bdata, PFN_DOWN(start_off) + merge,
  469. PFN_UP(end_off), BOOTMEM_EXCLUSIVE))
  470. BUG();
  471. region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
  472. start_off);
  473. memset(region, 0, size);
  474. /*
  475. * The min_count is set to 0 so that bootmem allocated blocks
  476. * are never reported as leaks.
  477. */
  478. kmemleak_alloc(region, size, 0, 0);
  479. return region;
  480. }
  481. if (fallback) {
  482. sidx = align_idx(bdata, fallback - 1, step);
  483. fallback = 0;
  484. goto find_block;
  485. }
  486. return NULL;
  487. }
  488. static void * __init alloc_bootmem_core(unsigned long size,
  489. unsigned long align,
  490. unsigned long goal,
  491. unsigned long limit)
  492. {
  493. bootmem_data_t *bdata;
  494. void *region;
  495. if (WARN_ON_ONCE(slab_is_available()))
  496. return kzalloc(size, GFP_NOWAIT);
  497. list_for_each_entry(bdata, &bdata_list, list) {
  498. if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
  499. continue;
  500. if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
  501. break;
  502. region = alloc_bootmem_bdata(bdata, size, align, goal, limit);
  503. if (region)
  504. return region;
  505. }
  506. return NULL;
  507. }
  508. static void * __init ___alloc_bootmem_nopanic(unsigned long size,
  509. unsigned long align,
  510. unsigned long goal,
  511. unsigned long limit)
  512. {
  513. void *ptr;
  514. restart:
  515. ptr = alloc_bootmem_core(size, align, goal, limit);
  516. if (ptr)
  517. return ptr;
  518. if (goal) {
  519. goal = 0;
  520. goto restart;
  521. }
  522. return NULL;
  523. }
  524. /**
  525. * __alloc_bootmem_nopanic - allocate boot memory without panicking
  526. * @size: size of the request in bytes
  527. * @align: alignment of the region
  528. * @goal: preferred starting address of the region
  529. *
  530. * The goal is dropped if it can not be satisfied and the allocation will
  531. * fall back to memory below @goal.
  532. *
  533. * Allocation may happen on any node in the system.
  534. *
  535. * Returns NULL on failure.
  536. */
  537. void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
  538. unsigned long goal)
  539. {
  540. unsigned long limit = 0;
  541. return ___alloc_bootmem_nopanic(size, align, goal, limit);
  542. }
  543. static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
  544. unsigned long goal, unsigned long limit)
  545. {
  546. void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
  547. if (mem)
  548. return mem;
  549. /*
  550. * Whoops, we cannot satisfy the allocation request.
  551. */
  552. printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
  553. panic("Out of memory");
  554. return NULL;
  555. }
  556. /**
  557. * __alloc_bootmem - allocate boot memory
  558. * @size: size of the request in bytes
  559. * @align: alignment of the region
  560. * @goal: preferred starting address of the region
  561. *
  562. * The goal is dropped if it can not be satisfied and the allocation will
  563. * fall back to memory below @goal.
  564. *
  565. * Allocation may happen on any node in the system.
  566. *
  567. * The function panics if the request can not be satisfied.
  568. */
  569. void * __init __alloc_bootmem(unsigned long size, unsigned long align,
  570. unsigned long goal)
  571. {
  572. unsigned long limit = 0;
  573. return ___alloc_bootmem(size, align, goal, limit);
  574. }
  575. void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
  576. unsigned long size, unsigned long align,
  577. unsigned long goal, unsigned long limit)
  578. {
  579. void *ptr;
  580. if (WARN_ON_ONCE(slab_is_available()))
  581. return kzalloc(size, GFP_NOWAIT);
  582. again:
  583. /* do not panic in alloc_bootmem_bdata() */
  584. if (limit && goal + size > limit)
  585. limit = 0;
  586. ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, goal, limit);
  587. if (ptr)
  588. return ptr;
  589. ptr = alloc_bootmem_core(size, align, goal, limit);
  590. if (ptr)
  591. return ptr;
  592. if (goal) {
  593. goal = 0;
  594. goto again;
  595. }
  596. return NULL;
  597. }
  598. void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
  599. unsigned long align, unsigned long goal)
  600. {
  601. if (WARN_ON_ONCE(slab_is_available()))
  602. return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
  603. return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
  604. }
  605. void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
  606. unsigned long align, unsigned long goal,
  607. unsigned long limit)
  608. {
  609. void *ptr;
  610. ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
  611. if (ptr)
  612. return ptr;
  613. printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
  614. panic("Out of memory");
  615. return NULL;
  616. }
  617. /**
  618. * __alloc_bootmem_node - allocate boot memory from a specific node
  619. * @pgdat: node to allocate from
  620. * @size: size of the request in bytes
  621. * @align: alignment of the region
  622. * @goal: preferred starting address of the region
  623. *
  624. * The goal is dropped if it can not be satisfied and the allocation will
  625. * fall back to memory below @goal.
  626. *
  627. * Allocation may fall back to any node in the system if the specified node
  628. * can not hold the requested memory.
  629. *
  630. * The function panics if the request can not be satisfied.
  631. */
  632. void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
  633. unsigned long align, unsigned long goal)
  634. {
  635. if (WARN_ON_ONCE(slab_is_available()))
  636. return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
  637. return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
  638. }
  639. void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
  640. unsigned long align, unsigned long goal)
  641. {
  642. #ifdef MAX_DMA32_PFN
  643. unsigned long end_pfn;
  644. if (WARN_ON_ONCE(slab_is_available()))
  645. return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
  646. /* update goal according ...MAX_DMA32_PFN */
  647. end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  648. if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
  649. (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
  650. void *ptr;
  651. unsigned long new_goal;
  652. new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
  653. ptr = alloc_bootmem_bdata(pgdat->bdata, size, align,
  654. new_goal, 0);
  655. if (ptr)
  656. return ptr;
  657. }
  658. #endif
  659. return __alloc_bootmem_node(pgdat, size, align, goal);
  660. }
  661. #ifndef ARCH_LOW_ADDRESS_LIMIT
  662. #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
  663. #endif
  664. /**
  665. * __alloc_bootmem_low - allocate low boot memory
  666. * @size: size of the request in bytes
  667. * @align: alignment of the region
  668. * @goal: preferred starting address of the region
  669. *
  670. * The goal is dropped if it can not be satisfied and the allocation will
  671. * fall back to memory below @goal.
  672. *
  673. * Allocation may happen on any node in the system.
  674. *
  675. * The function panics if the request can not be satisfied.
  676. */
  677. void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
  678. unsigned long goal)
  679. {
  680. return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
  681. }
  682. /**
  683. * __alloc_bootmem_low_node - allocate low boot memory from a specific node
  684. * @pgdat: node to allocate from
  685. * @size: size of the request in bytes
  686. * @align: alignment of the region
  687. * @goal: preferred starting address of the region
  688. *
  689. * The goal is dropped if it can not be satisfied and the allocation will
  690. * fall back to memory below @goal.
  691. *
  692. * Allocation may fall back to any node in the system if the specified node
  693. * can not hold the requested memory.
  694. *
  695. * The function panics if the request can not be satisfied.
  696. */
  697. void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
  698. unsigned long align, unsigned long goal)
  699. {
  700. if (WARN_ON_ONCE(slab_is_available()))
  701. return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
  702. return ___alloc_bootmem_node(pgdat, size, align,
  703. goal, ARCH_LOW_ADDRESS_LIMIT);
  704. }