bootmem.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793
  1. /*
  2. * bootmem - A boot-time physical memory allocator and configurator
  3. *
  4. * Copyright (C) 1999 Ingo Molnar
  5. * 1999 Kanoj Sarcar, SGI
  6. * 2008 Johannes Weiner
  7. *
  8. * Access to this subsystem has to be serialized externally (which is true
  9. * for the boot process anyway).
  10. */
  11. #include <linux/init.h>
  12. #include <linux/pfn.h>
  13. #include <linux/bootmem.h>
  14. #include <linux/module.h>
  15. #include <asm/bug.h>
  16. #include <asm/io.h>
  17. #include <asm/processor.h>
  18. #include "internal.h"
  19. unsigned long max_low_pfn;
  20. unsigned long min_low_pfn;
  21. unsigned long max_pfn;
  22. static LIST_HEAD(bdata_list);
  23. #ifdef CONFIG_CRASH_DUMP
  24. /*
  25. * If we have booted due to a crash, max_pfn will be a very low value. We need
  26. * to know the amount of memory that the previous kernel used.
  27. */
  28. unsigned long saved_max_pfn;
  29. #endif
  30. bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
  31. static int bootmem_debug;
  32. static int __init bootmem_debug_setup(char *buf)
  33. {
  34. bootmem_debug = 1;
  35. return 0;
  36. }
  37. early_param("bootmem_debug", bootmem_debug_setup);
  38. #define bdebug(fmt, args...) ({ \
  39. if (unlikely(bootmem_debug)) \
  40. printk(KERN_INFO \
  41. "bootmem::%s " fmt, \
  42. __FUNCTION__, ## args); \
  43. })
  44. static unsigned long __init bootmap_bytes(unsigned long pages)
  45. {
  46. unsigned long bytes = (pages + 7) / 8;
  47. return ALIGN(bytes, sizeof(long));
  48. }
  49. /**
  50. * bootmem_bootmap_pages - calculate bitmap size in pages
  51. * @pages: number of pages the bitmap has to represent
  52. */
  53. unsigned long __init bootmem_bootmap_pages(unsigned long pages)
  54. {
  55. unsigned long bytes = bootmap_bytes(pages);
  56. return PAGE_ALIGN(bytes) >> PAGE_SHIFT;
  57. }
  58. /*
  59. * link bdata in order
  60. */
  61. static void __init link_bootmem(bootmem_data_t *bdata)
  62. {
  63. bootmem_data_t *ent;
  64. if (list_empty(&bdata_list)) {
  65. list_add(&bdata->list, &bdata_list);
  66. return;
  67. }
  68. /* insert in order */
  69. list_for_each_entry(ent, &bdata_list, list) {
  70. if (bdata->node_boot_start < ent->node_boot_start) {
  71. list_add_tail(&bdata->list, &ent->list);
  72. return;
  73. }
  74. }
  75. list_add_tail(&bdata->list, &bdata_list);
  76. }
  77. /*
  78. * Called once to set up the allocator itself.
  79. */
  80. static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
  81. unsigned long mapstart, unsigned long start, unsigned long end)
  82. {
  83. unsigned long mapsize;
  84. mminit_validate_memmodel_limits(&start, &end);
  85. bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
  86. bdata->node_boot_start = PFN_PHYS(start);
  87. bdata->node_low_pfn = end;
  88. link_bootmem(bdata);
  89. /*
  90. * Initially all pages are reserved - setup_arch() has to
  91. * register free RAM areas explicitly.
  92. */
  93. mapsize = bootmap_bytes(end - start);
  94. memset(bdata->node_bootmem_map, 0xff, mapsize);
  95. bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx\n",
  96. bdata - bootmem_node_data, start, mapstart, end, mapsize);
  97. return mapsize;
  98. }
  99. /**
  100. * init_bootmem_node - register a node as boot memory
  101. * @pgdat: node to register
  102. * @freepfn: pfn where the bitmap for this node is to be placed
  103. * @startpfn: first pfn on the node
  104. * @endpfn: first pfn after the node
  105. *
  106. * Returns the number of bytes needed to hold the bitmap for this node.
  107. */
  108. unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
  109. unsigned long startpfn, unsigned long endpfn)
  110. {
  111. return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
  112. }
  113. /**
  114. * init_bootmem - register boot memory
  115. * @start: pfn where the bitmap is to be placed
  116. * @pages: number of available physical pages
  117. *
  118. * Returns the number of bytes needed to hold the bitmap.
  119. */
  120. unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
  121. {
  122. max_low_pfn = pages;
  123. min_low_pfn = start;
  124. return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
  125. }
  126. static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
  127. {
  128. struct page *page;
  129. unsigned long pfn;
  130. unsigned long i, count;
  131. unsigned long idx, pages;
  132. unsigned long *map;
  133. int gofast = 0;
  134. BUG_ON(!bdata->node_bootmem_map);
  135. count = 0;
  136. /* first extant page of the node */
  137. pfn = PFN_DOWN(bdata->node_boot_start);
  138. idx = bdata->node_low_pfn - pfn;
  139. map = bdata->node_bootmem_map;
  140. /*
  141. * Check if we are aligned to BITS_PER_LONG pages. If so, we might
  142. * be able to free page orders of that size at once.
  143. */
  144. if (!(pfn & (BITS_PER_LONG-1)))
  145. gofast = 1;
  146. for (i = 0; i < idx; ) {
  147. unsigned long v = ~map[i / BITS_PER_LONG];
  148. if (gofast && v == ~0UL) {
  149. int order;
  150. page = pfn_to_page(pfn);
  151. count += BITS_PER_LONG;
  152. order = ffs(BITS_PER_LONG) - 1;
  153. __free_pages_bootmem(page, order);
  154. i += BITS_PER_LONG;
  155. page += BITS_PER_LONG;
  156. } else if (v) {
  157. unsigned long m;
  158. page = pfn_to_page(pfn);
  159. for (m = 1; m && i < idx; m<<=1, page++, i++) {
  160. if (v & m) {
  161. count++;
  162. __free_pages_bootmem(page, 0);
  163. }
  164. }
  165. } else {
  166. i += BITS_PER_LONG;
  167. }
  168. pfn += BITS_PER_LONG;
  169. }
  170. /*
  171. * Now free the allocator bitmap itself, it's not
  172. * needed anymore:
  173. */
  174. page = virt_to_page(bdata->node_bootmem_map);
  175. pages = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
  176. idx = bootmem_bootmap_pages(pages);
  177. for (i = 0; i < idx; i++, page++)
  178. __free_pages_bootmem(page, 0);
  179. count += i;
  180. bdata->node_bootmem_map = NULL;
  181. bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);
  182. return count;
  183. }
  184. /**
  185. * free_all_bootmem_node - release a node's free pages to the buddy allocator
  186. * @pgdat: node to be released
  187. *
  188. * Returns the number of pages actually released.
  189. */
  190. unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
  191. {
  192. register_page_bootmem_info_node(pgdat);
  193. return free_all_bootmem_core(pgdat->bdata);
  194. }
  195. /**
  196. * free_all_bootmem - release free pages to the buddy allocator
  197. *
  198. * Returns the number of pages actually released.
  199. */
  200. unsigned long __init free_all_bootmem(void)
  201. {
  202. return free_all_bootmem_core(NODE_DATA(0)->bdata);
  203. }
  204. static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
  205. unsigned long size)
  206. {
  207. unsigned long sidx, eidx;
  208. unsigned long i;
  209. BUG_ON(!size);
  210. /* out range */
  211. if (addr + size < bdata->node_boot_start ||
  212. PFN_DOWN(addr) > bdata->node_low_pfn)
  213. return;
  214. /*
  215. * round down end of usable mem, partially free pages are
  216. * considered reserved.
  217. */
  218. if (addr >= bdata->node_boot_start && addr < bdata->last_success)
  219. bdata->last_success = addr;
  220. /*
  221. * Round up to index to the range.
  222. */
  223. if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
  224. sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
  225. else
  226. sidx = 0;
  227. eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
  228. if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
  229. eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
  230. bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,
  231. sidx + PFN_DOWN(bdata->node_boot_start),
  232. eidx + PFN_DOWN(bdata->node_boot_start));
  233. for (i = sidx; i < eidx; i++) {
  234. if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
  235. BUG();
  236. }
  237. }
  238. /**
  239. * free_bootmem_node - mark a page range as usable
  240. * @pgdat: node the range resides on
  241. * @physaddr: starting address of the range
  242. * @size: size of the range in bytes
  243. *
  244. * Partial pages will be considered reserved and left as they are.
  245. *
  246. * Only physical pages that actually reside on @pgdat are marked.
  247. */
  248. void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
  249. unsigned long size)
  250. {
  251. free_bootmem_core(pgdat->bdata, physaddr, size);
  252. }
  253. /**
  254. * free_bootmem - mark a page range as usable
  255. * @addr: starting address of the range
  256. * @size: size of the range in bytes
  257. *
  258. * Partial pages will be considered reserved and left as they are.
  259. *
  260. * All physical pages within the range are marked, no matter what
  261. * node they reside on.
  262. */
  263. void __init free_bootmem(unsigned long addr, unsigned long size)
  264. {
  265. bootmem_data_t *bdata;
  266. list_for_each_entry(bdata, &bdata_list, list)
  267. free_bootmem_core(bdata, addr, size);
  268. }
  269. /*
  270. * Marks a particular physical memory range as unallocatable. Usable RAM
  271. * might be used for boot-time allocations - or it might get added
  272. * to the free page pool later on.
  273. */
  274. static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
  275. unsigned long addr, unsigned long size, int flags)
  276. {
  277. unsigned long sidx, eidx;
  278. unsigned long i;
  279. BUG_ON(!size);
  280. /* out of range, don't hold other */
  281. if (addr + size < bdata->node_boot_start ||
  282. PFN_DOWN(addr) > bdata->node_low_pfn)
  283. return 0;
  284. /*
  285. * Round up to index to the range.
  286. */
  287. if (addr > bdata->node_boot_start)
  288. sidx= PFN_DOWN(addr - bdata->node_boot_start);
  289. else
  290. sidx = 0;
  291. eidx = PFN_UP(addr + size - bdata->node_boot_start);
  292. if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
  293. eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
  294. for (i = sidx; i < eidx; i++) {
  295. if (test_bit(i, bdata->node_bootmem_map)) {
  296. if (flags & BOOTMEM_EXCLUSIVE)
  297. return -EBUSY;
  298. }
  299. }
  300. return 0;
  301. }
  302. static void __init reserve_bootmem_core(bootmem_data_t *bdata,
  303. unsigned long addr, unsigned long size, int flags)
  304. {
  305. unsigned long sidx, eidx;
  306. unsigned long i;
  307. BUG_ON(!size);
  308. /* out of range */
  309. if (addr + size < bdata->node_boot_start ||
  310. PFN_DOWN(addr) > bdata->node_low_pfn)
  311. return;
  312. /*
  313. * Round up to index to the range.
  314. */
  315. if (addr > bdata->node_boot_start)
  316. sidx= PFN_DOWN(addr - bdata->node_boot_start);
  317. else
  318. sidx = 0;
  319. eidx = PFN_UP(addr + size - bdata->node_boot_start);
  320. if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
  321. eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
  322. bdebug("nid=%td start=%lx end=%lx flags=%x\n",
  323. bdata - bootmem_node_data,
  324. sidx + PFN_DOWN(bdata->node_boot_start),
  325. eidx + PFN_DOWN(bdata->node_boot_start),
  326. flags);
  327. for (i = sidx; i < eidx; i++)
  328. if (test_and_set_bit(i, bdata->node_bootmem_map))
  329. bdebug("hm, page %lx reserved twice.\n",
  330. PFN_DOWN(bdata->node_boot_start) + i);
  331. }
  332. /**
  333. * reserve_bootmem_node - mark a page range as reserved
  334. * @pgdat: node the range resides on
  335. * @physaddr: starting address of the range
  336. * @size: size of the range in bytes
  337. * @flags: reservation flags (see linux/bootmem.h)
  338. *
  339. * Partial pages will be reserved.
  340. *
  341. * Only physical pages that actually reside on @pgdat are marked.
  342. */
  343. int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
  344. unsigned long size, int flags)
  345. {
  346. int ret;
  347. ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
  348. if (ret < 0)
  349. return -ENOMEM;
  350. reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
  351. return 0;
  352. }
  353. #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
  354. /**
  355. * reserve_bootmem - mark a page range as usable
  356. * @addr: starting address of the range
  357. * @size: size of the range in bytes
  358. * @flags: reservation flags (see linux/bootmem.h)
  359. *
  360. * Partial pages will be reserved.
  361. *
  362. * All physical pages within the range are marked, no matter what
  363. * node they reside on.
  364. */
  365. int __init reserve_bootmem(unsigned long addr, unsigned long size,
  366. int flags)
  367. {
  368. bootmem_data_t *bdata;
  369. int ret;
  370. list_for_each_entry(bdata, &bdata_list, list) {
  371. ret = can_reserve_bootmem_core(bdata, addr, size, flags);
  372. if (ret < 0)
  373. return ret;
  374. }
  375. list_for_each_entry(bdata, &bdata_list, list)
  376. reserve_bootmem_core(bdata, addr, size, flags);
  377. return 0;
  378. }
  379. #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
  380. /*
  381. * We 'merge' subsequent allocations to save space. We might 'lose'
  382. * some fraction of a page if allocations cannot be satisfied due to
  383. * size constraints on boxes where there is physical RAM space
  384. * fragmentation - in these cases (mostly large memory boxes) this
  385. * is not a problem.
  386. *
  387. * On low memory boxes we get it right in 100% of the cases.
  388. *
  389. * alignment has to be a power of 2 value.
  390. *
  391. * NOTE: This function is _not_ reentrant.
  392. */
  393. static void * __init
  394. alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
  395. unsigned long align, unsigned long goal, unsigned long limit)
  396. {
  397. unsigned long areasize, preferred;
  398. unsigned long i, start = 0, incr, eidx, end_pfn;
  399. void *ret;
  400. unsigned long node_boot_start;
  401. void *node_bootmem_map;
  402. if (!size) {
  403. printk("alloc_bootmem_core(): zero-sized request\n");
  404. BUG();
  405. }
  406. BUG_ON(align & (align-1));
  407. /* on nodes without memory - bootmem_map is NULL */
  408. if (!bdata->node_bootmem_map)
  409. return NULL;
  410. bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
  411. bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
  412. align, goal, limit);
  413. /* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
  414. node_boot_start = bdata->node_boot_start;
  415. node_bootmem_map = bdata->node_bootmem_map;
  416. if (align) {
  417. node_boot_start = ALIGN(bdata->node_boot_start, align);
  418. if (node_boot_start > bdata->node_boot_start)
  419. node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
  420. PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
  421. }
  422. if (limit && node_boot_start >= limit)
  423. return NULL;
  424. end_pfn = bdata->node_low_pfn;
  425. limit = PFN_DOWN(limit);
  426. if (limit && end_pfn > limit)
  427. end_pfn = limit;
  428. eidx = end_pfn - PFN_DOWN(node_boot_start);
  429. /*
  430. * We try to allocate bootmem pages above 'goal'
  431. * first, then we try to allocate lower pages.
  432. */
  433. preferred = 0;
  434. if (goal && PFN_DOWN(goal) < end_pfn) {
  435. if (goal > node_boot_start)
  436. preferred = goal - node_boot_start;
  437. if (bdata->last_success > node_boot_start &&
  438. bdata->last_success - node_boot_start >= preferred)
  439. if (!limit || (limit && limit > bdata->last_success))
  440. preferred = bdata->last_success - node_boot_start;
  441. }
  442. preferred = PFN_DOWN(ALIGN(preferred, align));
  443. areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
  444. incr = align >> PAGE_SHIFT ? : 1;
  445. restart_scan:
  446. for (i = preferred; i < eidx;) {
  447. unsigned long j;
  448. i = find_next_zero_bit(node_bootmem_map, eidx, i);
  449. i = ALIGN(i, incr);
  450. if (i >= eidx)
  451. break;
  452. if (test_bit(i, node_bootmem_map)) {
  453. i += incr;
  454. continue;
  455. }
  456. for (j = i + 1; j < i + areasize; ++j) {
  457. if (j >= eidx)
  458. goto fail_block;
  459. if (test_bit(j, node_bootmem_map))
  460. goto fail_block;
  461. }
  462. start = i;
  463. goto found;
  464. fail_block:
  465. i = ALIGN(j, incr);
  466. if (i == j)
  467. i += incr;
  468. }
  469. if (preferred > 0) {
  470. preferred = 0;
  471. goto restart_scan;
  472. }
  473. return NULL;
  474. found:
  475. bdata->last_success = PFN_PHYS(start) + node_boot_start;
  476. BUG_ON(start >= eidx);
  477. /*
  478. * Is the next page of the previous allocation-end the start
  479. * of this allocation's buffer? If yes then we can 'merge'
  480. * the previous partial page with this allocation.
  481. */
  482. if (align < PAGE_SIZE &&
  483. bdata->last_offset && bdata->last_pos+1 == start) {
  484. unsigned long offset, remaining_size;
  485. offset = ALIGN(bdata->last_offset, align);
  486. BUG_ON(offset > PAGE_SIZE);
  487. remaining_size = PAGE_SIZE - offset;
  488. if (size < remaining_size) {
  489. areasize = 0;
  490. /* last_pos unchanged */
  491. bdata->last_offset = offset + size;
  492. ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
  493. offset + node_boot_start);
  494. } else {
  495. remaining_size = size - remaining_size;
  496. areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
  497. ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
  498. offset + node_boot_start);
  499. bdata->last_pos = start + areasize - 1;
  500. bdata->last_offset = remaining_size;
  501. }
  502. bdata->last_offset &= ~PAGE_MASK;
  503. } else {
  504. bdata->last_pos = start + areasize - 1;
  505. bdata->last_offset = size & ~PAGE_MASK;
  506. ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
  507. }
  508. bdebug("nid=%td start=%lx end=%lx\n",
  509. bdata - bootmem_node_data,
  510. start + PFN_DOWN(bdata->node_boot_start),
  511. start + areasize + PFN_DOWN(bdata->node_boot_start));
  512. /*
  513. * Reserve the area now:
  514. */
  515. for (i = start; i < start + areasize; i++)
  516. if (unlikely(test_and_set_bit(i, node_bootmem_map)))
  517. BUG();
  518. memset(ret, 0, size);
  519. return ret;
  520. }
  521. /**
  522. * __alloc_bootmem_nopanic - allocate boot memory without panicking
  523. * @size: size of the request in bytes
  524. * @align: alignment of the region
  525. * @goal: preferred starting address of the region
  526. *
  527. * The goal is dropped if it can not be satisfied and the allocation will
  528. * fall back to memory below @goal.
  529. *
  530. * Allocation may happen on any node in the system.
  531. *
  532. * Returns NULL on failure.
  533. */
  534. void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
  535. unsigned long goal)
  536. {
  537. bootmem_data_t *bdata;
  538. void *ptr;
  539. list_for_each_entry(bdata, &bdata_list, list) {
  540. ptr = alloc_bootmem_core(bdata, size, align, goal, 0);
  541. if (ptr)
  542. return ptr;
  543. }
  544. return NULL;
  545. }
  546. /**
  547. * __alloc_bootmem - allocate boot memory
  548. * @size: size of the request in bytes
  549. * @align: alignment of the region
  550. * @goal: preferred starting address of the region
  551. *
  552. * The goal is dropped if it can not be satisfied and the allocation will
  553. * fall back to memory below @goal.
  554. *
  555. * Allocation may happen on any node in the system.
  556. *
  557. * The function panics if the request can not be satisfied.
  558. */
  559. void * __init __alloc_bootmem(unsigned long size, unsigned long align,
  560. unsigned long goal)
  561. {
  562. void *mem = __alloc_bootmem_nopanic(size,align,goal);
  563. if (mem)
  564. return mem;
  565. /*
  566. * Whoops, we cannot satisfy the allocation request.
  567. */
  568. printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
  569. panic("Out of memory");
  570. return NULL;
  571. }
  572. /**
  573. * __alloc_bootmem_node - allocate boot memory from a specific node
  574. * @pgdat: node to allocate from
  575. * @size: size of the request in bytes
  576. * @align: alignment of the region
  577. * @goal: preferred starting address of the region
  578. *
  579. * The goal is dropped if it can not be satisfied and the allocation will
  580. * fall back to memory below @goal.
  581. *
  582. * Allocation may fall back to any node in the system if the specified node
  583. * can not hold the requested memory.
  584. *
  585. * The function panics if the request can not be satisfied.
  586. */
  587. void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
  588. unsigned long align, unsigned long goal)
  589. {
  590. void *ptr;
  591. ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
  592. if (ptr)
  593. return ptr;
  594. return __alloc_bootmem(size, align, goal);
  595. }
  596. #ifdef CONFIG_SPARSEMEM
  597. /**
  598. * alloc_bootmem_section - allocate boot memory from a specific section
  599. * @size: size of the request in bytes
  600. * @section_nr: sparse map section to allocate from
  601. *
  602. * Return NULL on failure.
  603. */
  604. void * __init alloc_bootmem_section(unsigned long size,
  605. unsigned long section_nr)
  606. {
  607. void *ptr;
  608. unsigned long limit, goal, start_nr, end_nr, pfn;
  609. struct pglist_data *pgdat;
  610. pfn = section_nr_to_pfn(section_nr);
  611. goal = PFN_PHYS(pfn);
  612. limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
  613. pgdat = NODE_DATA(early_pfn_to_nid(pfn));
  614. ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
  615. limit);
  616. if (!ptr)
  617. return NULL;
  618. start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
  619. end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
  620. if (start_nr != section_nr || end_nr != section_nr) {
  621. printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
  622. section_nr);
  623. free_bootmem_core(pgdat->bdata, __pa(ptr), size);
  624. ptr = NULL;
  625. }
  626. return ptr;
  627. }
  628. #endif
  629. void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
  630. unsigned long align, unsigned long goal)
  631. {
  632. void *ptr;
  633. ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
  634. if (ptr)
  635. return ptr;
  636. return __alloc_bootmem_nopanic(size, align, goal);
  637. }
  638. #ifndef ARCH_LOW_ADDRESS_LIMIT
  639. #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
  640. #endif
  641. /**
  642. * __alloc_bootmem_low - allocate low boot memory
  643. * @size: size of the request in bytes
  644. * @align: alignment of the region
  645. * @goal: preferred starting address of the region
  646. *
  647. * The goal is dropped if it can not be satisfied and the allocation will
  648. * fall back to memory below @goal.
  649. *
  650. * Allocation may happen on any node in the system.
  651. *
  652. * The function panics if the request can not be satisfied.
  653. */
  654. void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
  655. unsigned long goal)
  656. {
  657. bootmem_data_t *bdata;
  658. void *ptr;
  659. list_for_each_entry(bdata, &bdata_list, list) {
  660. ptr = alloc_bootmem_core(bdata, size, align, goal,
  661. ARCH_LOW_ADDRESS_LIMIT);
  662. if (ptr)
  663. return ptr;
  664. }
  665. /*
  666. * Whoops, we cannot satisfy the allocation request.
  667. */
  668. printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
  669. panic("Out of low memory");
  670. return NULL;
  671. }
  672. /**
  673. * __alloc_bootmem_low_node - allocate low boot memory from a specific node
  674. * @pgdat: node to allocate from
  675. * @size: size of the request in bytes
  676. * @align: alignment of the region
  677. * @goal: preferred starting address of the region
  678. *
  679. * The goal is dropped if it can not be satisfied and the allocation will
  680. * fall back to memory below @goal.
  681. *
  682. * Allocation may fall back to any node in the system if the specified node
  683. * can not hold the requested memory.
  684. *
  685. * The function panics if the request can not be satisfied.
  686. */
  687. void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
  688. unsigned long align, unsigned long goal)
  689. {
  690. return alloc_bootmem_core(pgdat->bdata, size, align, goal,
  691. ARCH_LOW_ADDRESS_LIMIT);
  692. }