numa_64.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080
  1. /*
  2. * Generic VM initialization for x86-64 NUMA setups.
  3. * Copyright 2002,2003 Andi Kleen, SuSE Labs.
  4. */
  5. #include <linux/kernel.h>
  6. #include <linux/mm.h>
  7. #include <linux/string.h>
  8. #include <linux/init.h>
  9. #include <linux/bootmem.h>
  10. #include <linux/memblock.h>
  11. #include <linux/mmzone.h>
  12. #include <linux/ctype.h>
  13. #include <linux/module.h>
  14. #include <linux/nodemask.h>
  15. #include <linux/sched.h>
  16. #include <linux/acpi.h>
  17. #include <asm/e820.h>
  18. #include <asm/proto.h>
  19. #include <asm/dma.h>
  20. #include <asm/numa.h>
  21. #include <asm/acpi.h>
  22. #include <asm/amd_nb.h>
  23. struct numa_memblk {
  24. u64 start;
  25. u64 end;
  26. int nid;
  27. };
  28. struct numa_meminfo {
  29. int nr_blks;
  30. struct numa_memblk blk[NR_NODE_MEMBLKS];
  31. };
  32. struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
  33. EXPORT_SYMBOL(node_data);
  34. nodemask_t numa_nodes_parsed __initdata;
  35. struct memnode memnode;
  36. static unsigned long __initdata nodemap_addr;
  37. static unsigned long __initdata nodemap_size;
  38. static struct numa_meminfo numa_meminfo __initdata;
  39. static int numa_distance_cnt;
  40. static u8 *numa_distance;
  41. #ifdef CONFIG_NUMA_EMU
  42. static bool numa_emu_dist;
  43. #endif
  44. /*
  45. * Given a shift value, try to populate memnodemap[]
  46. * Returns :
  47. * 1 if OK
  48. * 0 if memnodmap[] too small (of shift too small)
  49. * -1 if node overlap or lost ram (shift too big)
  50. */
  51. static int __init populate_memnodemap(const struct numa_meminfo *mi, int shift)
  52. {
  53. unsigned long addr, end;
  54. int i, res = -1;
  55. memset(memnodemap, 0xff, sizeof(s16)*memnodemapsize);
  56. for (i = 0; i < mi->nr_blks; i++) {
  57. addr = mi->blk[i].start;
  58. end = mi->blk[i].end;
  59. if (addr >= end)
  60. continue;
  61. if ((end >> shift) >= memnodemapsize)
  62. return 0;
  63. do {
  64. if (memnodemap[addr >> shift] != NUMA_NO_NODE)
  65. return -1;
  66. memnodemap[addr >> shift] = mi->blk[i].nid;
  67. addr += (1UL << shift);
  68. } while (addr < end);
  69. res = 1;
  70. }
  71. return res;
  72. }
  73. static int __init allocate_cachealigned_memnodemap(void)
  74. {
  75. unsigned long addr;
  76. memnodemap = memnode.embedded_map;
  77. if (memnodemapsize <= ARRAY_SIZE(memnode.embedded_map))
  78. return 0;
  79. addr = 0x8000;
  80. nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
  81. nodemap_addr = memblock_find_in_range(addr, get_max_mapped(),
  82. nodemap_size, L1_CACHE_BYTES);
  83. if (nodemap_addr == MEMBLOCK_ERROR) {
  84. printk(KERN_ERR
  85. "NUMA: Unable to allocate Memory to Node hash map\n");
  86. nodemap_addr = nodemap_size = 0;
  87. return -1;
  88. }
  89. memnodemap = phys_to_virt(nodemap_addr);
  90. memblock_x86_reserve_range(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP");
  91. printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
  92. nodemap_addr, nodemap_addr + nodemap_size);
  93. return 0;
  94. }
  95. /*
  96. * The LSB of all start and end addresses in the node map is the value of the
  97. * maximum possible shift.
  98. */
  99. static int __init extract_lsb_from_nodes(const struct numa_meminfo *mi)
  100. {
  101. int i, nodes_used = 0;
  102. unsigned long start, end;
  103. unsigned long bitfield = 0, memtop = 0;
  104. for (i = 0; i < mi->nr_blks; i++) {
  105. start = mi->blk[i].start;
  106. end = mi->blk[i].end;
  107. if (start >= end)
  108. continue;
  109. bitfield |= start;
  110. nodes_used++;
  111. if (end > memtop)
  112. memtop = end;
  113. }
  114. if (nodes_used <= 1)
  115. i = 63;
  116. else
  117. i = find_first_bit(&bitfield, sizeof(unsigned long)*8);
  118. memnodemapsize = (memtop >> i)+1;
  119. return i;
  120. }
  121. static int __init compute_hash_shift(const struct numa_meminfo *mi)
  122. {
  123. int shift;
  124. shift = extract_lsb_from_nodes(mi);
  125. if (allocate_cachealigned_memnodemap())
  126. return -1;
  127. printk(KERN_DEBUG "NUMA: Using %d for the hash shift.\n",
  128. shift);
  129. if (populate_memnodemap(mi, shift) != 1) {
  130. printk(KERN_INFO "Your memory is not aligned you need to "
  131. "rebuild your kernel with a bigger NODEMAPSIZE "
  132. "shift=%d\n", shift);
  133. return -1;
  134. }
  135. return shift;
  136. }
  137. int __meminit __early_pfn_to_nid(unsigned long pfn)
  138. {
  139. return phys_to_nid(pfn << PAGE_SHIFT);
  140. }
  141. static void * __init early_node_mem(int nodeid, unsigned long start,
  142. unsigned long end, unsigned long size,
  143. unsigned long align)
  144. {
  145. unsigned long mem;
  146. /*
  147. * put it on high as possible
  148. * something will go with NODE_DATA
  149. */
  150. if (start < (MAX_DMA_PFN<<PAGE_SHIFT))
  151. start = MAX_DMA_PFN<<PAGE_SHIFT;
  152. if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) &&
  153. end > (MAX_DMA32_PFN<<PAGE_SHIFT))
  154. start = MAX_DMA32_PFN<<PAGE_SHIFT;
  155. mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align);
  156. if (mem != MEMBLOCK_ERROR)
  157. return __va(mem);
  158. /* extend the search scope */
  159. end = max_pfn_mapped << PAGE_SHIFT;
  160. start = MAX_DMA_PFN << PAGE_SHIFT;
  161. mem = memblock_find_in_range(start, end, size, align);
  162. if (mem != MEMBLOCK_ERROR)
  163. return __va(mem);
  164. printk(KERN_ERR "Cannot find %lu bytes in node %d\n",
  165. size, nodeid);
  166. return NULL;
  167. }
  168. static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
  169. struct numa_meminfo *mi)
  170. {
  171. /* ignore zero length blks */
  172. if (start == end)
  173. return 0;
  174. /* whine about and ignore invalid blks */
  175. if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
  176. pr_warning("NUMA: Warning: invalid memblk node %d (%Lx-%Lx)\n",
  177. nid, start, end);
  178. return 0;
  179. }
  180. if (mi->nr_blks >= NR_NODE_MEMBLKS) {
  181. pr_err("NUMA: too many memblk ranges\n");
  182. return -EINVAL;
  183. }
  184. mi->blk[mi->nr_blks].start = start;
  185. mi->blk[mi->nr_blks].end = end;
  186. mi->blk[mi->nr_blks].nid = nid;
  187. mi->nr_blks++;
  188. return 0;
  189. }
  190. static void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi)
  191. {
  192. mi->nr_blks--;
  193. memmove(&mi->blk[idx], &mi->blk[idx + 1],
  194. (mi->nr_blks - idx) * sizeof(mi->blk[0]));
  195. }
  196. int __init numa_add_memblk(int nid, u64 start, u64 end)
  197. {
  198. return numa_add_memblk_to(nid, start, end, &numa_meminfo);
  199. }
  200. /* Initialize bootmem allocator for a node */
  201. void __init
  202. setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
  203. {
  204. unsigned long start_pfn, last_pfn, nodedata_phys;
  205. const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
  206. int nid;
  207. if (!end)
  208. return;
  209. /*
  210. * Don't confuse VM with a node that doesn't have the
  211. * minimum amount of memory:
  212. */
  213. if (end && (end - start) < NODE_MIN_SIZE)
  214. return;
  215. start = roundup(start, ZONE_ALIGN);
  216. printk(KERN_INFO "Initmem setup node %d %016lx-%016lx\n", nodeid,
  217. start, end);
  218. start_pfn = start >> PAGE_SHIFT;
  219. last_pfn = end >> PAGE_SHIFT;
  220. node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size,
  221. SMP_CACHE_BYTES);
  222. if (node_data[nodeid] == NULL)
  223. return;
  224. nodedata_phys = __pa(node_data[nodeid]);
  225. memblock_x86_reserve_range(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA");
  226. printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys,
  227. nodedata_phys + pgdat_size - 1);
  228. nid = phys_to_nid(nodedata_phys);
  229. if (nid != nodeid)
  230. printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nodeid, nid);
  231. memset(NODE_DATA(nodeid), 0, sizeof(pg_data_t));
  232. NODE_DATA(nodeid)->node_id = nodeid;
  233. NODE_DATA(nodeid)->node_start_pfn = start_pfn;
  234. NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn;
  235. node_set_online(nodeid);
  236. }
  237. static int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
  238. {
  239. const u64 low = 0;
  240. const u64 high = (u64)max_pfn << PAGE_SHIFT;
  241. int i, j, k;
  242. for (i = 0; i < mi->nr_blks; i++) {
  243. struct numa_memblk *bi = &mi->blk[i];
  244. /* make sure all blocks are inside the limits */
  245. bi->start = max(bi->start, low);
  246. bi->end = min(bi->end, high);
  247. /* and there's no empty block */
  248. if (bi->start == bi->end) {
  249. numa_remove_memblk_from(i--, mi);
  250. continue;
  251. }
  252. for (j = i + 1; j < mi->nr_blks; j++) {
  253. struct numa_memblk *bj = &mi->blk[j];
  254. unsigned long start, end;
  255. /*
  256. * See whether there are overlapping blocks. Whine
  257. * about but allow overlaps of the same nid. They
  258. * will be merged below.
  259. */
  260. if (bi->end > bj->start && bi->start < bj->end) {
  261. if (bi->nid != bj->nid) {
  262. pr_err("NUMA: node %d (%Lx-%Lx) overlaps with node %d (%Lx-%Lx)\n",
  263. bi->nid, bi->start, bi->end,
  264. bj->nid, bj->start, bj->end);
  265. return -EINVAL;
  266. }
  267. pr_warning("NUMA: Warning: node %d (%Lx-%Lx) overlaps with itself (%Lx-%Lx)\n",
  268. bi->nid, bi->start, bi->end,
  269. bj->start, bj->end);
  270. }
  271. /*
  272. * Join together blocks on the same node, holes
  273. * between which don't overlap with memory on other
  274. * nodes.
  275. */
  276. if (bi->nid != bj->nid)
  277. continue;
  278. start = max(min(bi->start, bj->start), low);
  279. end = min(max(bi->end, bj->end), high);
  280. for (k = 0; k < mi->nr_blks; k++) {
  281. struct numa_memblk *bk = &mi->blk[k];
  282. if (bi->nid == bk->nid)
  283. continue;
  284. if (start < bk->end && end > bk->start)
  285. break;
  286. }
  287. if (k < mi->nr_blks)
  288. continue;
  289. printk(KERN_INFO "NUMA: Node %d [%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
  290. bi->nid, bi->start, bi->end, bj->start, bj->end,
  291. start, end);
  292. bi->start = start;
  293. bi->end = end;
  294. numa_remove_memblk_from(j--, mi);
  295. }
  296. }
  297. for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {
  298. mi->blk[i].start = mi->blk[i].end = 0;
  299. mi->blk[i].nid = NUMA_NO_NODE;
  300. }
  301. return 0;
  302. }
  303. /*
  304. * Set nodes, which have memory in @mi, in *@nodemask.
  305. */
  306. static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
  307. const struct numa_meminfo *mi)
  308. {
  309. int i;
  310. for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
  311. if (mi->blk[i].start != mi->blk[i].end &&
  312. mi->blk[i].nid != NUMA_NO_NODE)
  313. node_set(mi->blk[i].nid, *nodemask);
  314. }
  315. /*
  316. * Reset distance table. The current table is freed. The next
  317. * numa_set_distance() call will create a new one.
  318. */
  319. static void __init numa_reset_distance(void)
  320. {
  321. size_t size;
  322. size = numa_distance_cnt * sizeof(numa_distance[0]);
  323. memblock_x86_free_range(__pa(numa_distance),
  324. __pa(numa_distance) + size);
  325. numa_distance = NULL;
  326. numa_distance_cnt = 0;
  327. }
  328. /*
  329. * Set the distance between node @from to @to to @distance. If distance
  330. * table doesn't exist, one which is large enough to accomodate all the
  331. * currently known nodes will be created.
  332. */
  333. void __init numa_set_distance(int from, int to, int distance)
  334. {
  335. if (!numa_distance) {
  336. nodemask_t nodes_parsed;
  337. size_t size;
  338. int i, j, cnt = 0;
  339. u64 phys;
  340. /* size the new table and allocate it */
  341. nodes_parsed = numa_nodes_parsed;
  342. numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
  343. for_each_node_mask(i, nodes_parsed)
  344. cnt = i;
  345. size = ++cnt * sizeof(numa_distance[0]);
  346. phys = memblock_find_in_range(0,
  347. (u64)max_pfn_mapped << PAGE_SHIFT,
  348. size, PAGE_SIZE);
  349. if (phys == MEMBLOCK_ERROR) {
  350. pr_warning("NUMA: Warning: can't allocate distance table!\n");
  351. /* don't retry until explicitly reset */
  352. numa_distance = (void *)1LU;
  353. return;
  354. }
  355. memblock_x86_reserve_range(phys, phys + size, "NUMA DIST");
  356. numa_distance = __va(phys);
  357. numa_distance_cnt = cnt;
  358. /* fill with the default distances */
  359. for (i = 0; i < cnt; i++)
  360. for (j = 0; j < cnt; j++)
  361. numa_distance[i * cnt + j] = i == j ?
  362. LOCAL_DISTANCE : REMOTE_DISTANCE;
  363. printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt);
  364. }
  365. if (from >= numa_distance_cnt || to >= numa_distance_cnt) {
  366. printk_once(KERN_DEBUG "NUMA: Debug: distance out of bound, from=%d to=%d distance=%d\n",
  367. from, to, distance);
  368. return;
  369. }
  370. if ((u8)distance != distance ||
  371. (from == to && distance != LOCAL_DISTANCE)) {
  372. pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
  373. from, to, distance);
  374. return;
  375. }
  376. numa_distance[from * numa_distance_cnt + to] = distance;
  377. }
  378. int __node_distance(int from, int to)
  379. {
  380. #if defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA_EMU)
  381. if (numa_emu_dist)
  382. return acpi_emu_node_distance(from, to);
  383. #endif
  384. if (from >= numa_distance_cnt || to >= numa_distance_cnt)
  385. return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
  386. return numa_distance[from * numa_distance_cnt + to];
  387. }
  388. EXPORT_SYMBOL(__node_distance);
  389. /*
  390. * Sanity check to catch more bad NUMA configurations (they are amazingly
  391. * common). Make sure the nodes cover all memory.
  392. */
  393. static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
  394. {
  395. unsigned long numaram, e820ram;
  396. int i;
  397. numaram = 0;
  398. for (i = 0; i < mi->nr_blks; i++) {
  399. unsigned long s = mi->blk[i].start >> PAGE_SHIFT;
  400. unsigned long e = mi->blk[i].end >> PAGE_SHIFT;
  401. numaram += e - s;
  402. numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
  403. if ((long)numaram < 0)
  404. numaram = 0;
  405. }
  406. e820ram = max_pfn - (memblock_x86_hole_size(0,
  407. max_pfn << PAGE_SHIFT) >> PAGE_SHIFT);
  408. /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
  409. if ((long)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
  410. printk(KERN_ERR "NUMA: nodes only cover %luMB of your %luMB e820 RAM. Not used.\n",
  411. (numaram << PAGE_SHIFT) >> 20,
  412. (e820ram << PAGE_SHIFT) >> 20);
  413. return false;
  414. }
  415. return true;
  416. }
  417. static int __init numa_register_memblks(struct numa_meminfo *mi)
  418. {
  419. int i, j, nid;
  420. /* Account for nodes with cpus and no memory */
  421. node_possible_map = numa_nodes_parsed;
  422. numa_nodemask_from_meminfo(&node_possible_map, mi);
  423. if (WARN_ON(nodes_empty(node_possible_map)))
  424. return -EINVAL;
  425. memnode_shift = compute_hash_shift(mi);
  426. if (memnode_shift < 0) {
  427. printk(KERN_ERR "NUMA: No NUMA node hash function found. Contact maintainer\n");
  428. return -EINVAL;
  429. }
  430. for (i = 0; i < mi->nr_blks; i++)
  431. memblock_x86_register_active_regions(mi->blk[i].nid,
  432. mi->blk[i].start >> PAGE_SHIFT,
  433. mi->blk[i].end >> PAGE_SHIFT);
  434. /* for out of order entries */
  435. sort_node_map();
  436. if (!numa_meminfo_cover_memory(mi))
  437. return -EINVAL;
  438. init_memory_mapping_high();
  439. /*
  440. * Finally register nodes. Do it twice in case setup_node_bootmem
  441. * missed one due to missing bootmem.
  442. */
  443. for (i = 0; i < 2; i++) {
  444. for_each_node_mask(nid, node_possible_map) {
  445. u64 start = (u64)max_pfn << PAGE_SHIFT;
  446. u64 end = 0;
  447. if (node_online(nid))
  448. continue;
  449. for (j = 0; j < mi->nr_blks; j++) {
  450. if (nid != mi->blk[j].nid)
  451. continue;
  452. start = min(mi->blk[j].start, start);
  453. end = max(mi->blk[j].end, end);
  454. }
  455. if (start < end)
  456. setup_node_bootmem(nid, start, end);
  457. }
  458. }
  459. return 0;
  460. }
  461. #ifdef CONFIG_NUMA_EMU
  462. /* Numa emulation */
  463. static int emu_nid_to_phys[MAX_NUMNODES] __cpuinitdata;
  464. static char *emu_cmdline __initdata;
  465. void __init numa_emu_cmdline(char *str)
  466. {
  467. emu_cmdline = str;
  468. }
  469. static int __init emu_find_memblk_by_nid(int nid, const struct numa_meminfo *mi)
  470. {
  471. int i;
  472. for (i = 0; i < mi->nr_blks; i++)
  473. if (mi->blk[i].nid == nid)
  474. return i;
  475. return -ENOENT;
  476. }
  477. int __init find_node_by_addr(unsigned long addr)
  478. {
  479. const struct numa_meminfo *mi = &numa_meminfo;
  480. int i;
  481. for (i = 0; i < mi->nr_blks; i++) {
  482. /*
  483. * Find the real node that this emulated node appears on. For
  484. * the sake of simplicity, we only use a real node's starting
  485. * address to determine which emulated node it appears on.
  486. */
  487. if (addr >= mi->blk[i].start && addr < mi->blk[i].end)
  488. return mi->blk[i].nid;
  489. }
  490. return NUMA_NO_NODE;
  491. }
  492. static void __init fake_physnodes(int acpi, int amd,
  493. const struct numa_meminfo *ei)
  494. {
  495. static struct bootnode nodes[MAX_NUMNODES] __initdata;
  496. int i, nr_nodes = 0;
  497. for (i = 0; i < ei->nr_blks; i++) {
  498. int nid = ei->blk[i].nid;
  499. if (nodes[nid].start == nodes[nid].end) {
  500. nodes[nid].start = ei->blk[i].start;
  501. nodes[nid].end = ei->blk[i].end;
  502. nr_nodes++;
  503. } else {
  504. nodes[nid].start = min(ei->blk[i].start, nodes[nid].start);
  505. nodes[nid].end = max(ei->blk[i].end, nodes[nid].end);
  506. }
  507. }
  508. BUG_ON(acpi && amd);
  509. #ifdef CONFIG_ACPI_NUMA
  510. if (acpi)
  511. acpi_fake_nodes(nodes, nr_nodes);
  512. #endif
  513. #ifdef CONFIG_AMD_NUMA
  514. if (amd)
  515. amd_fake_nodes(nodes, nr_nodes);
  516. #endif
  517. if (!acpi && !amd)
  518. for (i = 0; i < nr_cpu_ids; i++)
  519. numa_set_node(i, 0);
  520. }
  521. /*
  522. * Sets up nid to range from @start to @end. The return value is -errno if
  523. * something went wrong, 0 otherwise.
  524. */
  525. static int __init emu_setup_memblk(struct numa_meminfo *ei,
  526. struct numa_meminfo *pi,
  527. int nid, int phys_blk, u64 size)
  528. {
  529. struct numa_memblk *eb = &ei->blk[ei->nr_blks];
  530. struct numa_memblk *pb = &pi->blk[phys_blk];
  531. if (ei->nr_blks >= NR_NODE_MEMBLKS) {
  532. pr_err("NUMA: Too many emulated memblks, failing emulation\n");
  533. return -EINVAL;
  534. }
  535. ei->nr_blks++;
  536. eb->start = pb->start;
  537. eb->end = pb->start + size;
  538. eb->nid = nid;
  539. if (emu_nid_to_phys[nid] == NUMA_NO_NODE)
  540. emu_nid_to_phys[nid] = pb->nid;
  541. pb->start += size;
  542. if (pb->start >= pb->end) {
  543. WARN_ON_ONCE(pb->start > pb->end);
  544. numa_remove_memblk_from(phys_blk, pi);
  545. }
  546. printk(KERN_INFO "Faking node %d at %016Lx-%016Lx (%LuMB)\n", nid,
  547. eb->start, eb->end, (eb->end - eb->start) >> 20);
  548. return 0;
  549. }
  550. /*
  551. * Sets up nr_nodes fake nodes interleaved over physical nodes ranging from addr
  552. * to max_addr. The return value is the number of nodes allocated.
  553. */
  554. static int __init split_nodes_interleave(struct numa_meminfo *ei,
  555. struct numa_meminfo *pi,
  556. u64 addr, u64 max_addr, int nr_nodes)
  557. {
  558. nodemask_t physnode_mask = NODE_MASK_NONE;
  559. u64 size;
  560. int big;
  561. int nid = 0;
  562. int i, ret;
  563. if (nr_nodes <= 0)
  564. return -1;
  565. if (nr_nodes > MAX_NUMNODES) {
  566. pr_info("numa=fake=%d too large, reducing to %d\n",
  567. nr_nodes, MAX_NUMNODES);
  568. nr_nodes = MAX_NUMNODES;
  569. }
  570. size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) / nr_nodes;
  571. /*
  572. * Calculate the number of big nodes that can be allocated as a result
  573. * of consolidating the remainder.
  574. */
  575. big = ((size & ~FAKE_NODE_MIN_HASH_MASK) * nr_nodes) /
  576. FAKE_NODE_MIN_SIZE;
  577. size &= FAKE_NODE_MIN_HASH_MASK;
  578. if (!size) {
  579. pr_err("Not enough memory for each node. "
  580. "NUMA emulation disabled.\n");
  581. return -1;
  582. }
  583. for (i = 0; i < pi->nr_blks; i++)
  584. node_set(pi->blk[i].nid, physnode_mask);
  585. /*
  586. * Continue to fill physical nodes with fake nodes until there is no
  587. * memory left on any of them.
  588. */
  589. while (nodes_weight(physnode_mask)) {
  590. for_each_node_mask(i, physnode_mask) {
  591. u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN);
  592. u64 start, limit, end;
  593. int phys_blk;
  594. phys_blk = emu_find_memblk_by_nid(i, pi);
  595. if (phys_blk < 0) {
  596. node_clear(i, physnode_mask);
  597. continue;
  598. }
  599. start = pi->blk[phys_blk].start;
  600. limit = pi->blk[phys_blk].end;
  601. end = start + size;
  602. if (nid < big)
  603. end += FAKE_NODE_MIN_SIZE;
  604. /*
  605. * Continue to add memory to this fake node if its
  606. * non-reserved memory is less than the per-node size.
  607. */
  608. while (end - start -
  609. memblock_x86_hole_size(start, end) < size) {
  610. end += FAKE_NODE_MIN_SIZE;
  611. if (end > limit) {
  612. end = limit;
  613. break;
  614. }
  615. }
  616. /*
  617. * If there won't be at least FAKE_NODE_MIN_SIZE of
  618. * non-reserved memory in ZONE_DMA32 for the next node,
  619. * this one must extend to the boundary.
  620. */
  621. if (end < dma32_end && dma32_end - end -
  622. memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
  623. end = dma32_end;
  624. /*
  625. * If there won't be enough non-reserved memory for the
  626. * next node, this one must extend to the end of the
  627. * physical node.
  628. */
  629. if (limit - end -
  630. memblock_x86_hole_size(end, limit) < size)
  631. end = limit;
  632. ret = emu_setup_memblk(ei, pi, nid++ % nr_nodes,
  633. phys_blk,
  634. min(end, limit) - start);
  635. if (ret < 0)
  636. return ret;
  637. }
  638. }
  639. return 0;
  640. }
  641. /*
  642. * Returns the end address of a node so that there is at least `size' amount of
  643. * non-reserved memory or `max_addr' is reached.
  644. */
  645. static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size)
  646. {
  647. u64 end = start + size;
  648. while (end - start - memblock_x86_hole_size(start, end) < size) {
  649. end += FAKE_NODE_MIN_SIZE;
  650. if (end > max_addr) {
  651. end = max_addr;
  652. break;
  653. }
  654. }
  655. return end;
  656. }
  657. /*
  658. * Sets up fake nodes of `size' interleaved over physical nodes ranging from
  659. * `addr' to `max_addr'. The return value is the number of nodes allocated.
  660. */
  661. static int __init split_nodes_size_interleave(struct numa_meminfo *ei,
  662. struct numa_meminfo *pi,
  663. u64 addr, u64 max_addr, u64 size)
  664. {
  665. nodemask_t physnode_mask = NODE_MASK_NONE;
  666. u64 min_size;
  667. int nid = 0;
  668. int i, ret;
  669. if (!size)
  670. return -1;
  671. /*
  672. * The limit on emulated nodes is MAX_NUMNODES, so the size per node is
  673. * increased accordingly if the requested size is too small. This
  674. * creates a uniform distribution of node sizes across the entire
  675. * machine (but not necessarily over physical nodes).
  676. */
  677. min_size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) /
  678. MAX_NUMNODES;
  679. min_size = max(min_size, FAKE_NODE_MIN_SIZE);
  680. if ((min_size & FAKE_NODE_MIN_HASH_MASK) < min_size)
  681. min_size = (min_size + FAKE_NODE_MIN_SIZE) &
  682. FAKE_NODE_MIN_HASH_MASK;
  683. if (size < min_size) {
  684. pr_err("Fake node size %LuMB too small, increasing to %LuMB\n",
  685. size >> 20, min_size >> 20);
  686. size = min_size;
  687. }
  688. size &= FAKE_NODE_MIN_HASH_MASK;
  689. for (i = 0; i < pi->nr_blks; i++)
  690. node_set(pi->blk[i].nid, physnode_mask);
  691. /*
  692. * Fill physical nodes with fake nodes of size until there is no memory
  693. * left on any of them.
  694. */
  695. while (nodes_weight(physnode_mask)) {
  696. for_each_node_mask(i, physnode_mask) {
  697. u64 dma32_end = MAX_DMA32_PFN << PAGE_SHIFT;
  698. u64 start, limit, end;
  699. int phys_blk;
  700. phys_blk = emu_find_memblk_by_nid(i, pi);
  701. if (phys_blk < 0) {
  702. node_clear(i, physnode_mask);
  703. continue;
  704. }
  705. start = pi->blk[phys_blk].start;
  706. limit = pi->blk[phys_blk].end;
  707. end = find_end_of_node(start, limit, size);
  708. /*
  709. * If there won't be at least FAKE_NODE_MIN_SIZE of
  710. * non-reserved memory in ZONE_DMA32 for the next node,
  711. * this one must extend to the boundary.
  712. */
  713. if (end < dma32_end && dma32_end - end -
  714. memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
  715. end = dma32_end;
  716. /*
  717. * If there won't be enough non-reserved memory for the
  718. * next node, this one must extend to the end of the
  719. * physical node.
  720. */
  721. if (limit - end -
  722. memblock_x86_hole_size(end, limit) < size)
  723. end = limit;
  724. ret = emu_setup_memblk(ei, pi, nid++ % MAX_NUMNODES,
  725. phys_blk,
  726. min(end, limit) - start);
  727. if (ret < 0)
  728. return ret;
  729. }
  730. }
  731. return 0;
  732. }
  733. /*
  734. * Sets up the system RAM area from start_pfn to last_pfn according to the
  735. * numa=fake command-line option.
  736. */
  737. static bool __init numa_emulation(int acpi, int amd)
  738. {
  739. static struct numa_meminfo ei __initdata;
  740. static struct numa_meminfo pi __initdata;
  741. const u64 max_addr = max_pfn << PAGE_SHIFT;
  742. int i, ret;
  743. memset(&ei, 0, sizeof(ei));
  744. pi = numa_meminfo;
  745. for (i = 0; i < MAX_NUMNODES; i++)
  746. emu_nid_to_phys[i] = NUMA_NO_NODE;
  747. /*
  748. * If the numa=fake command-line contains a 'M' or 'G', it represents
  749. * the fixed node size. Otherwise, if it is just a single number N,
  750. * split the system RAM into N fake nodes.
  751. */
  752. if (strchr(emu_cmdline, 'M') || strchr(emu_cmdline, 'G')) {
  753. u64 size;
  754. size = memparse(emu_cmdline, &emu_cmdline);
  755. ret = split_nodes_size_interleave(&ei, &pi, 0, max_addr, size);
  756. } else {
  757. unsigned long n;
  758. n = simple_strtoul(emu_cmdline, NULL, 0);
  759. ret = split_nodes_interleave(&ei, &pi, 0, max_addr, n);
  760. }
  761. if (ret < 0)
  762. return false;
  763. if (numa_cleanup_meminfo(&ei) < 0) {
  764. pr_warning("NUMA: Warning: constructed meminfo invalid, disabling emulation\n");
  765. return false;
  766. }
  767. /* commit */
  768. numa_meminfo = ei;
  769. /* make sure all emulated nodes are mapped to a physical node */
  770. for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++)
  771. if (emu_nid_to_phys[i] == NUMA_NO_NODE)
  772. emu_nid_to_phys[i] = 0;
  773. fake_physnodes(acpi, amd, &ei);
  774. numa_emu_dist = true;
  775. return true;
  776. }
  777. #endif /* CONFIG_NUMA_EMU */
  778. static int dummy_numa_init(void)
  779. {
  780. printk(KERN_INFO "%s\n",
  781. numa_off ? "NUMA turned off" : "No NUMA configuration found");
  782. printk(KERN_INFO "Faking a node at %016lx-%016lx\n",
  783. 0LU, max_pfn << PAGE_SHIFT);
  784. node_set(0, numa_nodes_parsed);
  785. numa_add_memblk(0, 0, (u64)max_pfn << PAGE_SHIFT);
  786. return 0;
  787. }
  788. void __init initmem_init(void)
  789. {
  790. int (*numa_init[])(void) = { [2] = dummy_numa_init };
  791. int i, j;
  792. if (!numa_off) {
  793. #ifdef CONFIG_ACPI_NUMA
  794. numa_init[0] = x86_acpi_numa_init;
  795. #endif
  796. #ifdef CONFIG_AMD_NUMA
  797. numa_init[1] = amd_numa_init;
  798. #endif
  799. }
  800. for (i = 0; i < ARRAY_SIZE(numa_init); i++) {
  801. if (!numa_init[i])
  802. continue;
  803. for (j = 0; j < MAX_LOCAL_APIC; j++)
  804. set_apicid_to_node(j, NUMA_NO_NODE);
  805. nodes_clear(numa_nodes_parsed);
  806. nodes_clear(node_possible_map);
  807. nodes_clear(node_online_map);
  808. memset(&numa_meminfo, 0, sizeof(numa_meminfo));
  809. remove_all_active_ranges();
  810. numa_reset_distance();
  811. if (numa_init[i]() < 0)
  812. continue;
  813. if (numa_cleanup_meminfo(&numa_meminfo) < 0)
  814. continue;
  815. #ifdef CONFIG_NUMA_EMU
  816. /*
  817. * If requested, try emulation. If emulation is not used,
  818. * build identity emu_nid_to_phys[] for numa_add_cpu()
  819. */
  820. if (!emu_cmdline || !numa_emulation(i == 0, i == 1))
  821. for (j = 0; j < ARRAY_SIZE(emu_nid_to_phys); j++)
  822. emu_nid_to_phys[j] = j;
  823. #endif
  824. if (numa_register_memblks(&numa_meminfo) < 0)
  825. continue;
  826. for (j = 0; j < nr_cpu_ids; j++) {
  827. int nid = early_cpu_to_node(j);
  828. if (nid == NUMA_NO_NODE)
  829. continue;
  830. if (!node_online(nid))
  831. numa_clear_node(j);
  832. }
  833. numa_init_array();
  834. return;
  835. }
  836. BUG();
  837. }
  838. unsigned long __init numa_free_all_bootmem(void)
  839. {
  840. unsigned long pages = 0;
  841. int i;
  842. for_each_online_node(i)
  843. pages += free_all_bootmem_node(NODE_DATA(i));
  844. pages += free_all_memory_core_early(MAX_NUMNODES);
  845. return pages;
  846. }
  847. int __cpuinit numa_cpu_node(int cpu)
  848. {
  849. int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
  850. if (apicid != BAD_APICID)
  851. return __apicid_to_node[apicid];
  852. return NUMA_NO_NODE;
  853. }
  854. /*
  855. * UGLINESS AHEAD: Currently, CONFIG_NUMA_EMU is 64bit only and makes use
  856. * of 64bit specific data structures. The distinction is artificial and
  857. * should be removed. numa_{add|remove}_cpu() are implemented in numa.c
  858. * for both 32 and 64bit when CONFIG_NUMA_EMU is disabled but here when
  859. * enabled.
  860. *
  861. * NUMA emulation is planned to be made generic and the following and other
  862. * related code should be moved to numa.c.
  863. */
  864. #ifdef CONFIG_NUMA_EMU
  865. # ifndef CONFIG_DEBUG_PER_CPU_MAPS
  866. void __cpuinit numa_add_cpu(int cpu)
  867. {
  868. int physnid, nid;
  869. nid = numa_cpu_node(cpu);
  870. if (nid == NUMA_NO_NODE)
  871. nid = early_cpu_to_node(cpu);
  872. BUG_ON(nid == NUMA_NO_NODE || !node_online(nid));
  873. physnid = emu_nid_to_phys[nid];
  874. /*
  875. * Map the cpu to each emulated node that is allocated on the physical
  876. * node of the cpu's apic id.
  877. */
  878. for_each_online_node(nid)
  879. if (emu_nid_to_phys[nid] == physnid)
  880. cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
  881. }
  882. void __cpuinit numa_remove_cpu(int cpu)
  883. {
  884. int i;
  885. for_each_online_node(i)
  886. cpumask_clear_cpu(cpu, node_to_cpumask_map[i]);
  887. }
  888. # else /* !CONFIG_DEBUG_PER_CPU_MAPS */
  889. static void __cpuinit numa_set_cpumask(int cpu, int enable)
  890. {
  891. struct cpumask *mask;
  892. int nid, physnid, i;
  893. nid = early_cpu_to_node(cpu);
  894. if (nid == NUMA_NO_NODE) {
  895. /* early_cpu_to_node() already emits a warning and trace */
  896. return;
  897. }
  898. physnid = emu_nid_to_phys[nid];
  899. for_each_online_node(i) {
  900. if (emu_nid_to_phys[nid] != physnid)
  901. continue;
  902. mask = debug_cpumask_set_cpu(cpu, enable);
  903. if (!mask)
  904. return;
  905. if (enable)
  906. cpumask_set_cpu(cpu, mask);
  907. else
  908. cpumask_clear_cpu(cpu, mask);
  909. }
  910. }
  911. void __cpuinit numa_add_cpu(int cpu)
  912. {
  913. numa_set_cpumask(cpu, 1);
  914. }
  915. void __cpuinit numa_remove_cpu(int cpu)
  916. {
  917. numa_set_cpumask(cpu, 0);
  918. }
  919. # endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
  920. #endif /* CONFIG_NUMA_EMU */