numa.c 40 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672
  1. /*
  2. * pSeries NUMA support
  3. *
  4. * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. #include <linux/threads.h>
  12. #include <linux/bootmem.h>
  13. #include <linux/init.h>
  14. #include <linux/mm.h>
  15. #include <linux/mmzone.h>
  16. #include <linux/export.h>
  17. #include <linux/nodemask.h>
  18. #include <linux/cpu.h>
  19. #include <linux/notifier.h>
  20. #include <linux/memblock.h>
  21. #include <linux/of.h>
  22. #include <linux/pfn.h>
  23. #include <linux/cpuset.h>
  24. #include <linux/node.h>
  25. #include <linux/stop_machine.h>
  26. #include <linux/proc_fs.h>
  27. #include <linux/seq_file.h>
  28. #include <linux/uaccess.h>
  29. #include <linux/slab.h>
  30. #include <asm/sparsemem.h>
  31. #include <asm/prom.h>
  32. #include <asm/smp.h>
  33. #include <asm/firmware.h>
  34. #include <asm/paca.h>
  35. #include <asm/hvcall.h>
  36. #include <asm/setup.h>
  37. #include <asm/vdso.h>
  38. static int numa_enabled = 1;
  39. static char *cmdline __initdata;
  40. static int numa_debug;
  41. #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
  42. int numa_cpu_lookup_table[NR_CPUS];
  43. cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
  44. struct pglist_data *node_data[MAX_NUMNODES];
  45. EXPORT_SYMBOL(numa_cpu_lookup_table);
  46. EXPORT_SYMBOL(node_to_cpumask_map);
  47. EXPORT_SYMBOL(node_data);
  48. static int min_common_depth;
  49. static int n_mem_addr_cells, n_mem_size_cells;
  50. static int form1_affinity;
  51. #define MAX_DISTANCE_REF_POINTS 4
  52. static int distance_ref_points_depth;
  53. static const unsigned int *distance_ref_points;
  54. static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS];
  55. /*
  56. * Allocate node_to_cpumask_map based on number of available nodes
  57. * Requires node_possible_map to be valid.
  58. *
  59. * Note: cpumask_of_node() is not valid until after this is done.
  60. */
  61. static void __init setup_node_to_cpumask_map(void)
  62. {
  63. unsigned int node;
  64. /* setup nr_node_ids if not done yet */
  65. if (nr_node_ids == MAX_NUMNODES)
  66. setup_nr_node_ids();
  67. /* allocate the map */
  68. for (node = 0; node < nr_node_ids; node++)
  69. alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
  70. /* cpumask_of_node() will now work */
  71. dbg("Node to cpumask map for %d nodes\n", nr_node_ids);
  72. }
  73. static int __init fake_numa_create_new_node(unsigned long end_pfn,
  74. unsigned int *nid)
  75. {
  76. unsigned long long mem;
  77. char *p = cmdline;
  78. static unsigned int fake_nid;
  79. static unsigned long long curr_boundary;
  80. /*
  81. * Modify node id, iff we started creating NUMA nodes
  82. * We want to continue from where we left of the last time
  83. */
  84. if (fake_nid)
  85. *nid = fake_nid;
  86. /*
  87. * In case there are no more arguments to parse, the
  88. * node_id should be the same as the last fake node id
  89. * (we've handled this above).
  90. */
  91. if (!p)
  92. return 0;
  93. mem = memparse(p, &p);
  94. if (!mem)
  95. return 0;
  96. if (mem < curr_boundary)
  97. return 0;
  98. curr_boundary = mem;
  99. if ((end_pfn << PAGE_SHIFT) > mem) {
  100. /*
  101. * Skip commas and spaces
  102. */
  103. while (*p == ',' || *p == ' ' || *p == '\t')
  104. p++;
  105. cmdline = p;
  106. fake_nid++;
  107. *nid = fake_nid;
  108. dbg("created new fake_node with id %d\n", fake_nid);
  109. return 1;
  110. }
  111. return 0;
  112. }
  113. /*
  114. * get_node_active_region - Return active region containing pfn
  115. * Active range returned is empty if none found.
  116. * @pfn: The page to return the region for
  117. * @node_ar: Returned set to the active region containing @pfn
  118. */
  119. static void __init get_node_active_region(unsigned long pfn,
  120. struct node_active_region *node_ar)
  121. {
  122. unsigned long start_pfn, end_pfn;
  123. int i, nid;
  124. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  125. if (pfn >= start_pfn && pfn < end_pfn) {
  126. node_ar->nid = nid;
  127. node_ar->start_pfn = start_pfn;
  128. node_ar->end_pfn = end_pfn;
  129. break;
  130. }
  131. }
  132. }
  133. static void map_cpu_to_node(int cpu, int node)
  134. {
  135. numa_cpu_lookup_table[cpu] = node;
  136. dbg("adding cpu %d to node %d\n", cpu, node);
  137. if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node])))
  138. cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
  139. }
  140. #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR)
  141. static void unmap_cpu_from_node(unsigned long cpu)
  142. {
  143. int node = numa_cpu_lookup_table[cpu];
  144. dbg("removing cpu %lu from node %d\n", cpu, node);
  145. if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) {
  146. cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
  147. } else {
  148. printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
  149. cpu, node);
  150. }
  151. }
  152. #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */
  153. /* must hold reference to node during call */
  154. static const int *of_get_associativity(struct device_node *dev)
  155. {
  156. return of_get_property(dev, "ibm,associativity", NULL);
  157. }
  158. /*
  159. * Returns the property linux,drconf-usable-memory if
  160. * it exists (the property exists only in kexec/kdump kernels,
  161. * added by kexec-tools)
  162. */
  163. static const u32 *of_get_usable_memory(struct device_node *memory)
  164. {
  165. const u32 *prop;
  166. u32 len;
  167. prop = of_get_property(memory, "linux,drconf-usable-memory", &len);
  168. if (!prop || len < sizeof(unsigned int))
  169. return 0;
  170. return prop;
  171. }
  172. int __node_distance(int a, int b)
  173. {
  174. int i;
  175. int distance = LOCAL_DISTANCE;
  176. if (!form1_affinity)
  177. return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE);
  178. for (i = 0; i < distance_ref_points_depth; i++) {
  179. if (distance_lookup_table[a][i] == distance_lookup_table[b][i])
  180. break;
  181. /* Double the distance for each NUMA level */
  182. distance *= 2;
  183. }
  184. return distance;
  185. }
  186. static void initialize_distance_lookup_table(int nid,
  187. const unsigned int *associativity)
  188. {
  189. int i;
  190. if (!form1_affinity)
  191. return;
  192. for (i = 0; i < distance_ref_points_depth; i++) {
  193. distance_lookup_table[nid][i] =
  194. associativity[distance_ref_points[i]];
  195. }
  196. }
  197. /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
  198. * info is found.
  199. */
  200. static int associativity_to_nid(const unsigned int *associativity)
  201. {
  202. int nid = -1;
  203. if (min_common_depth == -1)
  204. goto out;
  205. if (associativity[0] >= min_common_depth)
  206. nid = associativity[min_common_depth];
  207. /* POWER4 LPAR uses 0xffff as invalid node */
  208. if (nid == 0xffff || nid >= MAX_NUMNODES)
  209. nid = -1;
  210. if (nid > 0 && associativity[0] >= distance_ref_points_depth)
  211. initialize_distance_lookup_table(nid, associativity);
  212. out:
  213. return nid;
  214. }
  215. /* Returns the nid associated with the given device tree node,
  216. * or -1 if not found.
  217. */
  218. static int of_node_to_nid_single(struct device_node *device)
  219. {
  220. int nid = -1;
  221. const unsigned int *tmp;
  222. tmp = of_get_associativity(device);
  223. if (tmp)
  224. nid = associativity_to_nid(tmp);
  225. return nid;
  226. }
  227. /* Walk the device tree upwards, looking for an associativity id */
  228. int of_node_to_nid(struct device_node *device)
  229. {
  230. struct device_node *tmp;
  231. int nid = -1;
  232. of_node_get(device);
  233. while (device) {
  234. nid = of_node_to_nid_single(device);
  235. if (nid != -1)
  236. break;
  237. tmp = device;
  238. device = of_get_parent(tmp);
  239. of_node_put(tmp);
  240. }
  241. of_node_put(device);
  242. return nid;
  243. }
  244. EXPORT_SYMBOL_GPL(of_node_to_nid);
  245. static int __init find_min_common_depth(void)
  246. {
  247. int depth;
  248. struct device_node *root;
  249. if (firmware_has_feature(FW_FEATURE_OPAL))
  250. root = of_find_node_by_path("/ibm,opal");
  251. else
  252. root = of_find_node_by_path("/rtas");
  253. if (!root)
  254. root = of_find_node_by_path("/");
  255. /*
  256. * This property is a set of 32-bit integers, each representing
  257. * an index into the ibm,associativity nodes.
  258. *
  259. * With form 0 affinity the first integer is for an SMP configuration
  260. * (should be all 0's) and the second is for a normal NUMA
  261. * configuration. We have only one level of NUMA.
  262. *
  263. * With form 1 affinity the first integer is the most significant
  264. * NUMA boundary and the following are progressively less significant
  265. * boundaries. There can be more than one level of NUMA.
  266. */
  267. distance_ref_points = of_get_property(root,
  268. "ibm,associativity-reference-points",
  269. &distance_ref_points_depth);
  270. if (!distance_ref_points) {
  271. dbg("NUMA: ibm,associativity-reference-points not found.\n");
  272. goto err;
  273. }
  274. distance_ref_points_depth /= sizeof(int);
  275. if (firmware_has_feature(FW_FEATURE_OPAL) ||
  276. firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) {
  277. dbg("Using form 1 affinity\n");
  278. form1_affinity = 1;
  279. }
  280. if (form1_affinity) {
  281. depth = distance_ref_points[0];
  282. } else {
  283. if (distance_ref_points_depth < 2) {
  284. printk(KERN_WARNING "NUMA: "
  285. "short ibm,associativity-reference-points\n");
  286. goto err;
  287. }
  288. depth = distance_ref_points[1];
  289. }
  290. /*
  291. * Warn and cap if the hardware supports more than
  292. * MAX_DISTANCE_REF_POINTS domains.
  293. */
  294. if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) {
  295. printk(KERN_WARNING "NUMA: distance array capped at "
  296. "%d entries\n", MAX_DISTANCE_REF_POINTS);
  297. distance_ref_points_depth = MAX_DISTANCE_REF_POINTS;
  298. }
  299. of_node_put(root);
  300. return depth;
  301. err:
  302. of_node_put(root);
  303. return -1;
  304. }
  305. static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
  306. {
  307. struct device_node *memory = NULL;
  308. memory = of_find_node_by_type(memory, "memory");
  309. if (!memory)
  310. panic("numa.c: No memory nodes found!");
  311. *n_addr_cells = of_n_addr_cells(memory);
  312. *n_size_cells = of_n_size_cells(memory);
  313. of_node_put(memory);
  314. }
  315. static unsigned long read_n_cells(int n, const unsigned int **buf)
  316. {
  317. unsigned long result = 0;
  318. while (n--) {
  319. result = (result << 32) | **buf;
  320. (*buf)++;
  321. }
  322. return result;
  323. }
  324. /*
  325. * Read the next memblock list entry from the ibm,dynamic-memory property
  326. * and return the information in the provided of_drconf_cell structure.
  327. */
  328. static void read_drconf_cell(struct of_drconf_cell *drmem, const u32 **cellp)
  329. {
  330. const u32 *cp;
  331. drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);
  332. cp = *cellp;
  333. drmem->drc_index = cp[0];
  334. drmem->reserved = cp[1];
  335. drmem->aa_index = cp[2];
  336. drmem->flags = cp[3];
  337. *cellp = cp + 4;
  338. }
  339. /*
  340. * Retrieve and validate the ibm,dynamic-memory property of the device tree.
  341. *
  342. * The layout of the ibm,dynamic-memory property is a number N of memblock
  343. * list entries followed by N memblock list entries. Each memblock list entry
  344. * contains information as laid out in the of_drconf_cell struct above.
  345. */
  346. static int of_get_drconf_memory(struct device_node *memory, const u32 **dm)
  347. {
  348. const u32 *prop;
  349. u32 len, entries;
  350. prop = of_get_property(memory, "ibm,dynamic-memory", &len);
  351. if (!prop || len < sizeof(unsigned int))
  352. return 0;
  353. entries = *prop++;
  354. /* Now that we know the number of entries, revalidate the size
  355. * of the property read in to ensure we have everything
  356. */
  357. if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
  358. return 0;
  359. *dm = prop;
  360. return entries;
  361. }
  362. /*
  363. * Retrieve and validate the ibm,lmb-size property for drconf memory
  364. * from the device tree.
  365. */
  366. static u64 of_get_lmb_size(struct device_node *memory)
  367. {
  368. const u32 *prop;
  369. u32 len;
  370. prop = of_get_property(memory, "ibm,lmb-size", &len);
  371. if (!prop || len < sizeof(unsigned int))
  372. return 0;
  373. return read_n_cells(n_mem_size_cells, &prop);
  374. }
  375. struct assoc_arrays {
  376. u32 n_arrays;
  377. u32 array_sz;
  378. const u32 *arrays;
  379. };
  380. /*
  381. * Retrieve and validate the list of associativity arrays for drconf
  382. * memory from the ibm,associativity-lookup-arrays property of the
  383. * device tree..
  384. *
  385. * The layout of the ibm,associativity-lookup-arrays property is a number N
  386. * indicating the number of associativity arrays, followed by a number M
  387. * indicating the size of each associativity array, followed by a list
  388. * of N associativity arrays.
  389. */
  390. static int of_get_assoc_arrays(struct device_node *memory,
  391. struct assoc_arrays *aa)
  392. {
  393. const u32 *prop;
  394. u32 len;
  395. prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
  396. if (!prop || len < 2 * sizeof(unsigned int))
  397. return -1;
  398. aa->n_arrays = *prop++;
  399. aa->array_sz = *prop++;
  400. /* Now that we know the number of arrays and size of each array,
  401. * revalidate the size of the property read in.
  402. */
  403. if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
  404. return -1;
  405. aa->arrays = prop;
  406. return 0;
  407. }
  408. /*
  409. * This is like of_node_to_nid_single() for memory represented in the
  410. * ibm,dynamic-reconfiguration-memory node.
  411. */
  412. static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
  413. struct assoc_arrays *aa)
  414. {
  415. int default_nid = 0;
  416. int nid = default_nid;
  417. int index;
  418. if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
  419. !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
  420. drmem->aa_index < aa->n_arrays) {
  421. index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
  422. nid = aa->arrays[index];
  423. if (nid == 0xffff || nid >= MAX_NUMNODES)
  424. nid = default_nid;
  425. }
  426. return nid;
  427. }
  428. /*
  429. * Figure out to which domain a cpu belongs and stick it there.
  430. * Return the id of the domain used.
  431. */
  432. static int numa_setup_cpu(unsigned long lcpu)
  433. {
  434. int nid = 0;
  435. struct device_node *cpu = of_get_cpu_node(lcpu, NULL);
  436. if (!cpu) {
  437. WARN_ON(1);
  438. goto out;
  439. }
  440. nid = of_node_to_nid_single(cpu);
  441. if (nid < 0 || !node_online(nid))
  442. nid = first_online_node;
  443. out:
  444. map_cpu_to_node(lcpu, nid);
  445. of_node_put(cpu);
  446. return nid;
  447. }
  448. static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action,
  449. void *hcpu)
  450. {
  451. unsigned long lcpu = (unsigned long)hcpu;
  452. int ret = NOTIFY_DONE;
  453. switch (action) {
  454. case CPU_UP_PREPARE:
  455. case CPU_UP_PREPARE_FROZEN:
  456. numa_setup_cpu(lcpu);
  457. ret = NOTIFY_OK;
  458. break;
  459. #ifdef CONFIG_HOTPLUG_CPU
  460. case CPU_DEAD:
  461. case CPU_DEAD_FROZEN:
  462. case CPU_UP_CANCELED:
  463. case CPU_UP_CANCELED_FROZEN:
  464. unmap_cpu_from_node(lcpu);
  465. break;
  466. ret = NOTIFY_OK;
  467. #endif
  468. }
  469. return ret;
  470. }
  471. /*
  472. * Check and possibly modify a memory region to enforce the memory limit.
  473. *
  474. * Returns the size the region should have to enforce the memory limit.
  475. * This will either be the original value of size, a truncated value,
  476. * or zero. If the returned value of size is 0 the region should be
  477. * discarded as it lies wholly above the memory limit.
  478. */
  479. static unsigned long __init numa_enforce_memory_limit(unsigned long start,
  480. unsigned long size)
  481. {
  482. /*
  483. * We use memblock_end_of_DRAM() in here instead of memory_limit because
  484. * we've already adjusted it for the limit and it takes care of
  485. * having memory holes below the limit. Also, in the case of
  486. * iommu_is_off, memory_limit is not set but is implicitly enforced.
  487. */
  488. if (start + size <= memblock_end_of_DRAM())
  489. return size;
  490. if (start >= memblock_end_of_DRAM())
  491. return 0;
  492. return memblock_end_of_DRAM() - start;
  493. }
  494. /*
  495. * Reads the counter for a given entry in
  496. * linux,drconf-usable-memory property
  497. */
  498. static inline int __init read_usm_ranges(const u32 **usm)
  499. {
  500. /*
  501. * For each lmb in ibm,dynamic-memory a corresponding
  502. * entry in linux,drconf-usable-memory property contains
  503. * a counter followed by that many (base, size) duple.
  504. * read the counter from linux,drconf-usable-memory
  505. */
  506. return read_n_cells(n_mem_size_cells, usm);
  507. }
  508. /*
  509. * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
  510. * node. This assumes n_mem_{addr,size}_cells have been set.
  511. */
  512. static void __init parse_drconf_memory(struct device_node *memory)
  513. {
  514. const u32 *uninitialized_var(dm), *usm;
  515. unsigned int n, rc, ranges, is_kexec_kdump = 0;
  516. unsigned long lmb_size, base, size, sz;
  517. int nid;
  518. struct assoc_arrays aa = { .arrays = NULL };
  519. n = of_get_drconf_memory(memory, &dm);
  520. if (!n)
  521. return;
  522. lmb_size = of_get_lmb_size(memory);
  523. if (!lmb_size)
  524. return;
  525. rc = of_get_assoc_arrays(memory, &aa);
  526. if (rc)
  527. return;
  528. /* check if this is a kexec/kdump kernel */
  529. usm = of_get_usable_memory(memory);
  530. if (usm != NULL)
  531. is_kexec_kdump = 1;
  532. for (; n != 0; --n) {
  533. struct of_drconf_cell drmem;
  534. read_drconf_cell(&drmem, &dm);
  535. /* skip this block if the reserved bit is set in flags (0x80)
  536. or if the block is not assigned to this partition (0x8) */
  537. if ((drmem.flags & DRCONF_MEM_RESERVED)
  538. || !(drmem.flags & DRCONF_MEM_ASSIGNED))
  539. continue;
  540. base = drmem.base_addr;
  541. size = lmb_size;
  542. ranges = 1;
  543. if (is_kexec_kdump) {
  544. ranges = read_usm_ranges(&usm);
  545. if (!ranges) /* there are no (base, size) duple */
  546. continue;
  547. }
  548. do {
  549. if (is_kexec_kdump) {
  550. base = read_n_cells(n_mem_addr_cells, &usm);
  551. size = read_n_cells(n_mem_size_cells, &usm);
  552. }
  553. nid = of_drconf_to_nid_single(&drmem, &aa);
  554. fake_numa_create_new_node(
  555. ((base + size) >> PAGE_SHIFT),
  556. &nid);
  557. node_set_online(nid);
  558. sz = numa_enforce_memory_limit(base, size);
  559. if (sz)
  560. memblock_set_node(base, sz, nid);
  561. } while (--ranges);
  562. }
  563. }
  564. static int __init parse_numa_properties(void)
  565. {
  566. struct device_node *memory;
  567. int default_nid = 0;
  568. unsigned long i;
  569. if (numa_enabled == 0) {
  570. printk(KERN_WARNING "NUMA disabled by user\n");
  571. return -1;
  572. }
  573. min_common_depth = find_min_common_depth();
  574. if (min_common_depth < 0)
  575. return min_common_depth;
  576. dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
  577. /*
  578. * Even though we connect cpus to numa domains later in SMP
  579. * init, we need to know the node ids now. This is because
  580. * each node to be onlined must have NODE_DATA etc backing it.
  581. */
  582. for_each_present_cpu(i) {
  583. struct device_node *cpu;
  584. int nid;
  585. cpu = of_get_cpu_node(i, NULL);
  586. BUG_ON(!cpu);
  587. nid = of_node_to_nid_single(cpu);
  588. of_node_put(cpu);
  589. /*
  590. * Don't fall back to default_nid yet -- we will plug
  591. * cpus into nodes once the memory scan has discovered
  592. * the topology.
  593. */
  594. if (nid < 0)
  595. continue;
  596. node_set_online(nid);
  597. }
  598. get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
  599. for_each_node_by_type(memory, "memory") {
  600. unsigned long start;
  601. unsigned long size;
  602. int nid;
  603. int ranges;
  604. const unsigned int *memcell_buf;
  605. unsigned int len;
  606. memcell_buf = of_get_property(memory,
  607. "linux,usable-memory", &len);
  608. if (!memcell_buf || len <= 0)
  609. memcell_buf = of_get_property(memory, "reg", &len);
  610. if (!memcell_buf || len <= 0)
  611. continue;
  612. /* ranges in cell */
  613. ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
  614. new_range:
  615. /* these are order-sensitive, and modify the buffer pointer */
  616. start = read_n_cells(n_mem_addr_cells, &memcell_buf);
  617. size = read_n_cells(n_mem_size_cells, &memcell_buf);
  618. /*
  619. * Assumption: either all memory nodes or none will
  620. * have associativity properties. If none, then
  621. * everything goes to default_nid.
  622. */
  623. nid = of_node_to_nid_single(memory);
  624. if (nid < 0)
  625. nid = default_nid;
  626. fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
  627. node_set_online(nid);
  628. if (!(size = numa_enforce_memory_limit(start, size))) {
  629. if (--ranges)
  630. goto new_range;
  631. else
  632. continue;
  633. }
  634. memblock_set_node(start, size, nid);
  635. if (--ranges)
  636. goto new_range;
  637. }
  638. /*
  639. * Now do the same thing for each MEMBLOCK listed in the
  640. * ibm,dynamic-memory property in the
  641. * ibm,dynamic-reconfiguration-memory node.
  642. */
  643. memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
  644. if (memory)
  645. parse_drconf_memory(memory);
  646. return 0;
  647. }
  648. static void __init setup_nonnuma(void)
  649. {
  650. unsigned long top_of_ram = memblock_end_of_DRAM();
  651. unsigned long total_ram = memblock_phys_mem_size();
  652. unsigned long start_pfn, end_pfn;
  653. unsigned int nid = 0;
  654. struct memblock_region *reg;
  655. printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
  656. top_of_ram, total_ram);
  657. printk(KERN_DEBUG "Memory hole size: %ldMB\n",
  658. (top_of_ram - total_ram) >> 20);
  659. for_each_memblock(memory, reg) {
  660. start_pfn = memblock_region_memory_base_pfn(reg);
  661. end_pfn = memblock_region_memory_end_pfn(reg);
  662. fake_numa_create_new_node(end_pfn, &nid);
  663. memblock_set_node(PFN_PHYS(start_pfn),
  664. PFN_PHYS(end_pfn - start_pfn), nid);
  665. node_set_online(nid);
  666. }
  667. }
  668. void __init dump_numa_cpu_topology(void)
  669. {
  670. unsigned int node;
  671. unsigned int cpu, count;
  672. if (min_common_depth == -1 || !numa_enabled)
  673. return;
  674. for_each_online_node(node) {
  675. printk(KERN_DEBUG "Node %d CPUs:", node);
  676. count = 0;
  677. /*
  678. * If we used a CPU iterator here we would miss printing
  679. * the holes in the cpumap.
  680. */
  681. for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
  682. if (cpumask_test_cpu(cpu,
  683. node_to_cpumask_map[node])) {
  684. if (count == 0)
  685. printk(" %u", cpu);
  686. ++count;
  687. } else {
  688. if (count > 1)
  689. printk("-%u", cpu - 1);
  690. count = 0;
  691. }
  692. }
  693. if (count > 1)
  694. printk("-%u", nr_cpu_ids - 1);
  695. printk("\n");
  696. }
  697. }
  698. static void __init dump_numa_memory_topology(void)
  699. {
  700. unsigned int node;
  701. unsigned int count;
  702. if (min_common_depth == -1 || !numa_enabled)
  703. return;
  704. for_each_online_node(node) {
  705. unsigned long i;
  706. printk(KERN_DEBUG "Node %d Memory:", node);
  707. count = 0;
  708. for (i = 0; i < memblock_end_of_DRAM();
  709. i += (1 << SECTION_SIZE_BITS)) {
  710. if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
  711. if (count == 0)
  712. printk(" 0x%lx", i);
  713. ++count;
  714. } else {
  715. if (count > 0)
  716. printk("-0x%lx", i);
  717. count = 0;
  718. }
  719. }
  720. if (count > 0)
  721. printk("-0x%lx", i);
  722. printk("\n");
  723. }
  724. }
  725. /*
  726. * Allocate some memory, satisfying the memblock or bootmem allocator where
  727. * required. nid is the preferred node and end is the physical address of
  728. * the highest address in the node.
  729. *
  730. * Returns the virtual address of the memory.
  731. */
  732. static void __init *careful_zallocation(int nid, unsigned long size,
  733. unsigned long align,
  734. unsigned long end_pfn)
  735. {
  736. void *ret;
  737. int new_nid;
  738. unsigned long ret_paddr;
  739. ret_paddr = __memblock_alloc_base(size, align, end_pfn << PAGE_SHIFT);
  740. /* retry over all memory */
  741. if (!ret_paddr)
  742. ret_paddr = __memblock_alloc_base(size, align, memblock_end_of_DRAM());
  743. if (!ret_paddr)
  744. panic("numa.c: cannot allocate %lu bytes for node %d",
  745. size, nid);
  746. ret = __va(ret_paddr);
  747. /*
  748. * We initialize the nodes in numeric order: 0, 1, 2...
  749. * and hand over control from the MEMBLOCK allocator to the
  750. * bootmem allocator. If this function is called for
  751. * node 5, then we know that all nodes <5 are using the
  752. * bootmem allocator instead of the MEMBLOCK allocator.
  753. *
  754. * So, check the nid from which this allocation came
  755. * and double check to see if we need to use bootmem
  756. * instead of the MEMBLOCK. We don't free the MEMBLOCK memory
  757. * since it would be useless.
  758. */
  759. new_nid = early_pfn_to_nid(ret_paddr >> PAGE_SHIFT);
  760. if (new_nid < nid) {
  761. ret = __alloc_bootmem_node(NODE_DATA(new_nid),
  762. size, align, 0);
  763. dbg("alloc_bootmem %p %lx\n", ret, size);
  764. }
  765. memset(ret, 0, size);
  766. return ret;
  767. }
  768. static struct notifier_block ppc64_numa_nb = {
  769. .notifier_call = cpu_numa_callback,
  770. .priority = 1 /* Must run before sched domains notifier. */
  771. };
  772. static void __init mark_reserved_regions_for_nid(int nid)
  773. {
  774. struct pglist_data *node = NODE_DATA(nid);
  775. struct memblock_region *reg;
  776. for_each_memblock(reserved, reg) {
  777. unsigned long physbase = reg->base;
  778. unsigned long size = reg->size;
  779. unsigned long start_pfn = physbase >> PAGE_SHIFT;
  780. unsigned long end_pfn = PFN_UP(physbase + size);
  781. struct node_active_region node_ar;
  782. unsigned long node_end_pfn = node->node_start_pfn +
  783. node->node_spanned_pages;
  784. /*
  785. * Check to make sure that this memblock.reserved area is
  786. * within the bounds of the node that we care about.
  787. * Checking the nid of the start and end points is not
  788. * sufficient because the reserved area could span the
  789. * entire node.
  790. */
  791. if (end_pfn <= node->node_start_pfn ||
  792. start_pfn >= node_end_pfn)
  793. continue;
  794. get_node_active_region(start_pfn, &node_ar);
  795. while (start_pfn < end_pfn &&
  796. node_ar.start_pfn < node_ar.end_pfn) {
  797. unsigned long reserve_size = size;
  798. /*
  799. * if reserved region extends past active region
  800. * then trim size to active region
  801. */
  802. if (end_pfn > node_ar.end_pfn)
  803. reserve_size = (node_ar.end_pfn << PAGE_SHIFT)
  804. - physbase;
  805. /*
  806. * Only worry about *this* node, others may not
  807. * yet have valid NODE_DATA().
  808. */
  809. if (node_ar.nid == nid) {
  810. dbg("reserve_bootmem %lx %lx nid=%d\n",
  811. physbase, reserve_size, node_ar.nid);
  812. reserve_bootmem_node(NODE_DATA(node_ar.nid),
  813. physbase, reserve_size,
  814. BOOTMEM_DEFAULT);
  815. }
  816. /*
  817. * if reserved region is contained in the active region
  818. * then done.
  819. */
  820. if (end_pfn <= node_ar.end_pfn)
  821. break;
  822. /*
  823. * reserved region extends past the active region
  824. * get next active region that contains this
  825. * reserved region
  826. */
  827. start_pfn = node_ar.end_pfn;
  828. physbase = start_pfn << PAGE_SHIFT;
  829. size = size - reserve_size;
  830. get_node_active_region(start_pfn, &node_ar);
  831. }
  832. }
  833. }
  834. void __init do_init_bootmem(void)
  835. {
  836. int nid;
  837. min_low_pfn = 0;
  838. max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
  839. max_pfn = max_low_pfn;
  840. if (parse_numa_properties())
  841. setup_nonnuma();
  842. else
  843. dump_numa_memory_topology();
  844. for_each_online_node(nid) {
  845. unsigned long start_pfn, end_pfn;
  846. void *bootmem_vaddr;
  847. unsigned long bootmap_pages;
  848. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  849. /*
  850. * Allocate the node structure node local if possible
  851. *
  852. * Be careful moving this around, as it relies on all
  853. * previous nodes' bootmem to be initialized and have
  854. * all reserved areas marked.
  855. */
  856. NODE_DATA(nid) = careful_zallocation(nid,
  857. sizeof(struct pglist_data),
  858. SMP_CACHE_BYTES, end_pfn);
  859. dbg("node %d\n", nid);
  860. dbg("NODE_DATA() = %p\n", NODE_DATA(nid));
  861. NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
  862. NODE_DATA(nid)->node_start_pfn = start_pfn;
  863. NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
  864. if (NODE_DATA(nid)->node_spanned_pages == 0)
  865. continue;
  866. dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
  867. dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
  868. bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
  869. bootmem_vaddr = careful_zallocation(nid,
  870. bootmap_pages << PAGE_SHIFT,
  871. PAGE_SIZE, end_pfn);
  872. dbg("bootmap_vaddr = %p\n", bootmem_vaddr);
  873. init_bootmem_node(NODE_DATA(nid),
  874. __pa(bootmem_vaddr) >> PAGE_SHIFT,
  875. start_pfn, end_pfn);
  876. free_bootmem_with_active_regions(nid, end_pfn);
  877. /*
  878. * Be very careful about moving this around. Future
  879. * calls to careful_zallocation() depend on this getting
  880. * done correctly.
  881. */
  882. mark_reserved_regions_for_nid(nid);
  883. sparse_memory_present_with_active_regions(nid);
  884. }
  885. init_bootmem_done = 1;
  886. /*
  887. * Now bootmem is initialised we can create the node to cpumask
  888. * lookup tables and setup the cpu callback to populate them.
  889. */
  890. setup_node_to_cpumask_map();
  891. register_cpu_notifier(&ppc64_numa_nb);
  892. cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
  893. (void *)(unsigned long)boot_cpuid);
  894. }
  895. void __init paging_init(void)
  896. {
  897. unsigned long max_zone_pfns[MAX_NR_ZONES];
  898. memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
  899. max_zone_pfns[ZONE_DMA] = memblock_end_of_DRAM() >> PAGE_SHIFT;
  900. free_area_init_nodes(max_zone_pfns);
  901. }
  902. static int __init early_numa(char *p)
  903. {
  904. if (!p)
  905. return 0;
  906. if (strstr(p, "off"))
  907. numa_enabled = 0;
  908. if (strstr(p, "debug"))
  909. numa_debug = 1;
  910. p = strstr(p, "fake=");
  911. if (p)
  912. cmdline = p + strlen("fake=");
  913. return 0;
  914. }
  915. early_param("numa", early_numa);
  916. #ifdef CONFIG_MEMORY_HOTPLUG
  917. /*
  918. * Find the node associated with a hot added memory section for
  919. * memory represented in the device tree by the property
  920. * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
  921. */
  922. static int hot_add_drconf_scn_to_nid(struct device_node *memory,
  923. unsigned long scn_addr)
  924. {
  925. const u32 *dm;
  926. unsigned int drconf_cell_cnt, rc;
  927. unsigned long lmb_size;
  928. struct assoc_arrays aa;
  929. int nid = -1;
  930. drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
  931. if (!drconf_cell_cnt)
  932. return -1;
  933. lmb_size = of_get_lmb_size(memory);
  934. if (!lmb_size)
  935. return -1;
  936. rc = of_get_assoc_arrays(memory, &aa);
  937. if (rc)
  938. return -1;
  939. for (; drconf_cell_cnt != 0; --drconf_cell_cnt) {
  940. struct of_drconf_cell drmem;
  941. read_drconf_cell(&drmem, &dm);
  942. /* skip this block if it is reserved or not assigned to
  943. * this partition */
  944. if ((drmem.flags & DRCONF_MEM_RESERVED)
  945. || !(drmem.flags & DRCONF_MEM_ASSIGNED))
  946. continue;
  947. if ((scn_addr < drmem.base_addr)
  948. || (scn_addr >= (drmem.base_addr + lmb_size)))
  949. continue;
  950. nid = of_drconf_to_nid_single(&drmem, &aa);
  951. break;
  952. }
  953. return nid;
  954. }
  955. /*
  956. * Find the node associated with a hot added memory section for memory
  957. * represented in the device tree as a node (i.e. memory@XXXX) for
  958. * each memblock.
  959. */
  960. int hot_add_node_scn_to_nid(unsigned long scn_addr)
  961. {
  962. struct device_node *memory;
  963. int nid = -1;
  964. for_each_node_by_type(memory, "memory") {
  965. unsigned long start, size;
  966. int ranges;
  967. const unsigned int *memcell_buf;
  968. unsigned int len;
  969. memcell_buf = of_get_property(memory, "reg", &len);
  970. if (!memcell_buf || len <= 0)
  971. continue;
  972. /* ranges in cell */
  973. ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
  974. while (ranges--) {
  975. start = read_n_cells(n_mem_addr_cells, &memcell_buf);
  976. size = read_n_cells(n_mem_size_cells, &memcell_buf);
  977. if ((scn_addr < start) || (scn_addr >= (start + size)))
  978. continue;
  979. nid = of_node_to_nid_single(memory);
  980. break;
  981. }
  982. if (nid >= 0)
  983. break;
  984. }
  985. of_node_put(memory);
  986. return nid;
  987. }
  988. /*
  989. * Find the node associated with a hot added memory section. Section
  990. * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that
  991. * sections are fully contained within a single MEMBLOCK.
  992. */
  993. int hot_add_scn_to_nid(unsigned long scn_addr)
  994. {
  995. struct device_node *memory = NULL;
  996. int nid, found = 0;
  997. if (!numa_enabled || (min_common_depth < 0))
  998. return first_online_node;
  999. memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
  1000. if (memory) {
  1001. nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
  1002. of_node_put(memory);
  1003. } else {
  1004. nid = hot_add_node_scn_to_nid(scn_addr);
  1005. }
  1006. if (nid < 0 || !node_online(nid))
  1007. nid = first_online_node;
  1008. if (NODE_DATA(nid)->node_spanned_pages)
  1009. return nid;
  1010. for_each_online_node(nid) {
  1011. if (NODE_DATA(nid)->node_spanned_pages) {
  1012. found = 1;
  1013. break;
  1014. }
  1015. }
  1016. BUG_ON(!found);
  1017. return nid;
  1018. }
  1019. static u64 hot_add_drconf_memory_max(void)
  1020. {
  1021. struct device_node *memory = NULL;
  1022. unsigned int drconf_cell_cnt = 0;
  1023. u64 lmb_size = 0;
  1024. const u32 *dm = 0;
  1025. memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
  1026. if (memory) {
  1027. drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
  1028. lmb_size = of_get_lmb_size(memory);
  1029. of_node_put(memory);
  1030. }
  1031. return lmb_size * drconf_cell_cnt;
  1032. }
  1033. /*
  1034. * memory_hotplug_max - return max address of memory that may be added
  1035. *
  1036. * This is currently only used on systems that support drconfig memory
  1037. * hotplug.
  1038. */
  1039. u64 memory_hotplug_max(void)
  1040. {
  1041. return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM());
  1042. }
  1043. #endif /* CONFIG_MEMORY_HOTPLUG */
  1044. /* Virtual Processor Home Node (VPHN) support */
  1045. #ifdef CONFIG_PPC_SPLPAR
  1046. struct topology_update_data {
  1047. struct topology_update_data *next;
  1048. unsigned int cpu;
  1049. int old_nid;
  1050. int new_nid;
  1051. };
  1052. static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS];
  1053. static cpumask_t cpu_associativity_changes_mask;
  1054. static int vphn_enabled;
  1055. static int prrn_enabled;
  1056. static void reset_topology_timer(void);
  1057. /*
  1058. * Store the current values of the associativity change counters in the
  1059. * hypervisor.
  1060. */
  1061. static void setup_cpu_associativity_change_counters(void)
  1062. {
  1063. int cpu;
  1064. /* The VPHN feature supports a maximum of 8 reference points */
  1065. BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8);
  1066. for_each_possible_cpu(cpu) {
  1067. int i;
  1068. u8 *counts = vphn_cpu_change_counts[cpu];
  1069. volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
  1070. for (i = 0; i < distance_ref_points_depth; i++)
  1071. counts[i] = hypervisor_counts[i];
  1072. }
  1073. }
  1074. /*
  1075. * The hypervisor maintains a set of 8 associativity change counters in
  1076. * the VPA of each cpu that correspond to the associativity levels in the
  1077. * ibm,associativity-reference-points property. When an associativity
  1078. * level changes, the corresponding counter is incremented.
  1079. *
  1080. * Set a bit in cpu_associativity_changes_mask for each cpu whose home
  1081. * node associativity levels have changed.
  1082. *
  1083. * Returns the number of cpus with unhandled associativity changes.
  1084. */
  1085. static int update_cpu_associativity_changes_mask(void)
  1086. {
  1087. int cpu;
  1088. cpumask_t *changes = &cpu_associativity_changes_mask;
  1089. for_each_possible_cpu(cpu) {
  1090. int i, changed = 0;
  1091. u8 *counts = vphn_cpu_change_counts[cpu];
  1092. volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
  1093. for (i = 0; i < distance_ref_points_depth; i++) {
  1094. if (hypervisor_counts[i] != counts[i]) {
  1095. counts[i] = hypervisor_counts[i];
  1096. changed = 1;
  1097. }
  1098. }
  1099. if (changed) {
  1100. cpumask_set_cpu(cpu, changes);
  1101. }
  1102. }
  1103. return cpumask_weight(changes);
  1104. }
  1105. /*
  1106. * 6 64-bit registers unpacked into 12 32-bit associativity values. To form
  1107. * the complete property we have to add the length in the first cell.
  1108. */
  1109. #define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1)
  1110. /*
  1111. * Convert the associativity domain numbers returned from the hypervisor
  1112. * to the sequence they would appear in the ibm,associativity property.
  1113. */
  1114. static int vphn_unpack_associativity(const long *packed, unsigned int *unpacked)
  1115. {
  1116. int i, nr_assoc_doms = 0;
  1117. const u16 *field = (const u16*) packed;
  1118. #define VPHN_FIELD_UNUSED (0xffff)
  1119. #define VPHN_FIELD_MSB (0x8000)
  1120. #define VPHN_FIELD_MASK (~VPHN_FIELD_MSB)
  1121. for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) {
  1122. if (*field == VPHN_FIELD_UNUSED) {
  1123. /* All significant fields processed, and remaining
  1124. * fields contain the reserved value of all 1's.
  1125. * Just store them.
  1126. */
  1127. unpacked[i] = *((u32*)field);
  1128. field += 2;
  1129. } else if (*field & VPHN_FIELD_MSB) {
  1130. /* Data is in the lower 15 bits of this field */
  1131. unpacked[i] = *field & VPHN_FIELD_MASK;
  1132. field++;
  1133. nr_assoc_doms++;
  1134. } else {
  1135. /* Data is in the lower 15 bits of this field
  1136. * concatenated with the next 16 bit field
  1137. */
  1138. unpacked[i] = *((u32*)field);
  1139. field += 2;
  1140. nr_assoc_doms++;
  1141. }
  1142. }
  1143. /* The first cell contains the length of the property */
  1144. unpacked[0] = nr_assoc_doms;
  1145. return nr_assoc_doms;
  1146. }
  1147. /*
  1148. * Retrieve the new associativity information for a virtual processor's
  1149. * home node.
  1150. */
  1151. static long hcall_vphn(unsigned long cpu, unsigned int *associativity)
  1152. {
  1153. long rc;
  1154. long retbuf[PLPAR_HCALL9_BUFSIZE] = {0};
  1155. u64 flags = 1;
  1156. int hwcpu = get_hard_smp_processor_id(cpu);
  1157. rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu);
  1158. vphn_unpack_associativity(retbuf, associativity);
  1159. return rc;
  1160. }
  1161. static long vphn_get_associativity(unsigned long cpu,
  1162. unsigned int *associativity)
  1163. {
  1164. long rc;
  1165. rc = hcall_vphn(cpu, associativity);
  1166. switch (rc) {
  1167. case H_FUNCTION:
  1168. printk(KERN_INFO
  1169. "VPHN is not supported. Disabling polling...\n");
  1170. stop_topology_update();
  1171. break;
  1172. case H_HARDWARE:
  1173. printk(KERN_ERR
  1174. "hcall_vphn() experienced a hardware fault "
  1175. "preventing VPHN. Disabling polling...\n");
  1176. stop_topology_update();
  1177. }
  1178. return rc;
  1179. }
  1180. /*
  1181. * Update the CPU maps and sysfs entries for a single CPU when its NUMA
  1182. * characteristics change. This function doesn't perform any locking and is
  1183. * only safe to call from stop_machine().
  1184. */
  1185. static int update_cpu_topology(void *data)
  1186. {
  1187. struct topology_update_data *update;
  1188. unsigned long cpu;
  1189. if (!data)
  1190. return -EINVAL;
  1191. cpu = get_cpu();
  1192. for (update = data; update; update = update->next) {
  1193. if (cpu != update->cpu)
  1194. continue;
  1195. unregister_cpu_under_node(update->cpu, update->old_nid);
  1196. unmap_cpu_from_node(update->cpu);
  1197. map_cpu_to_node(update->cpu, update->new_nid);
  1198. vdso_getcpu_init();
  1199. register_cpu_under_node(update->cpu, update->new_nid);
  1200. }
  1201. return 0;
  1202. }
  1203. /*
  1204. * Update the node maps and sysfs entries for each cpu whose home node
  1205. * has changed. Returns 1 when the topology has changed, and 0 otherwise.
  1206. */
  1207. int arch_update_cpu_topology(void)
  1208. {
  1209. unsigned int cpu, changed = 0;
  1210. struct topology_update_data *updates, *ud;
  1211. unsigned int associativity[VPHN_ASSOC_BUFSIZE] = {0};
  1212. cpumask_t updated_cpus;
  1213. struct device *dev;
  1214. int weight, i = 0;
  1215. weight = cpumask_weight(&cpu_associativity_changes_mask);
  1216. if (!weight)
  1217. return 0;
  1218. updates = kzalloc(weight * (sizeof(*updates)), GFP_KERNEL);
  1219. if (!updates)
  1220. return 0;
  1221. cpumask_clear(&updated_cpus);
  1222. for_each_cpu(cpu, &cpu_associativity_changes_mask) {
  1223. ud = &updates[i++];
  1224. ud->cpu = cpu;
  1225. vphn_get_associativity(cpu, associativity);
  1226. ud->new_nid = associativity_to_nid(associativity);
  1227. if (ud->new_nid < 0 || !node_online(ud->new_nid))
  1228. ud->new_nid = first_online_node;
  1229. ud->old_nid = numa_cpu_lookup_table[cpu];
  1230. cpumask_set_cpu(cpu, &updated_cpus);
  1231. if (i < weight)
  1232. ud->next = &updates[i];
  1233. }
  1234. stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
  1235. for (ud = &updates[0]; ud; ud = ud->next) {
  1236. dev = get_cpu_device(ud->cpu);
  1237. if (dev)
  1238. kobject_uevent(&dev->kobj, KOBJ_CHANGE);
  1239. cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask);
  1240. changed = 1;
  1241. }
  1242. kfree(updates);
  1243. return changed;
  1244. }
  1245. static void topology_work_fn(struct work_struct *work)
  1246. {
  1247. rebuild_sched_domains();
  1248. }
  1249. static DECLARE_WORK(topology_work, topology_work_fn);
  1250. void topology_schedule_update(void)
  1251. {
  1252. schedule_work(&topology_work);
  1253. }
  1254. static void topology_timer_fn(unsigned long ignored)
  1255. {
  1256. if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask))
  1257. topology_schedule_update();
  1258. else if (vphn_enabled) {
  1259. if (update_cpu_associativity_changes_mask() > 0)
  1260. topology_schedule_update();
  1261. reset_topology_timer();
  1262. }
  1263. }
  1264. static struct timer_list topology_timer =
  1265. TIMER_INITIALIZER(topology_timer_fn, 0, 0);
  1266. static void reset_topology_timer(void)
  1267. {
  1268. topology_timer.data = 0;
  1269. topology_timer.expires = jiffies + 60 * HZ;
  1270. mod_timer(&topology_timer, topology_timer.expires);
  1271. }
  1272. #ifdef CONFIG_SMP
  1273. static void stage_topology_update(int core_id)
  1274. {
  1275. cpumask_or(&cpu_associativity_changes_mask,
  1276. &cpu_associativity_changes_mask, cpu_sibling_mask(core_id));
  1277. reset_topology_timer();
  1278. }
  1279. static int dt_update_callback(struct notifier_block *nb,
  1280. unsigned long action, void *data)
  1281. {
  1282. struct of_prop_reconfig *update;
  1283. int rc = NOTIFY_DONE;
  1284. switch (action) {
  1285. case OF_RECONFIG_UPDATE_PROPERTY:
  1286. update = (struct of_prop_reconfig *)data;
  1287. if (!of_prop_cmp(update->dn->type, "cpu") &&
  1288. !of_prop_cmp(update->prop->name, "ibm,associativity")) {
  1289. u32 core_id;
  1290. of_property_read_u32(update->dn, "reg", &core_id);
  1291. stage_topology_update(core_id);
  1292. rc = NOTIFY_OK;
  1293. }
  1294. break;
  1295. }
  1296. return rc;
  1297. }
  1298. static struct notifier_block dt_update_nb = {
  1299. .notifier_call = dt_update_callback,
  1300. };
  1301. #endif
  1302. /*
  1303. * Start polling for associativity changes.
  1304. */
  1305. int start_topology_update(void)
  1306. {
  1307. int rc = 0;
  1308. if (firmware_has_feature(FW_FEATURE_PRRN)) {
  1309. if (!prrn_enabled) {
  1310. prrn_enabled = 1;
  1311. vphn_enabled = 0;
  1312. #ifdef CONFIG_SMP
  1313. rc = of_reconfig_notifier_register(&dt_update_nb);
  1314. #endif
  1315. }
  1316. } else if (firmware_has_feature(FW_FEATURE_VPHN) &&
  1317. get_lppaca()->shared_proc) {
  1318. if (!vphn_enabled) {
  1319. prrn_enabled = 0;
  1320. vphn_enabled = 1;
  1321. setup_cpu_associativity_change_counters();
  1322. init_timer_deferrable(&topology_timer);
  1323. reset_topology_timer();
  1324. }
  1325. }
  1326. return rc;
  1327. }
  1328. /*
  1329. * Disable polling for VPHN associativity changes.
  1330. */
  1331. int stop_topology_update(void)
  1332. {
  1333. int rc = 0;
  1334. if (prrn_enabled) {
  1335. prrn_enabled = 0;
  1336. #ifdef CONFIG_SMP
  1337. rc = of_reconfig_notifier_unregister(&dt_update_nb);
  1338. #endif
  1339. } else if (vphn_enabled) {
  1340. vphn_enabled = 0;
  1341. rc = del_timer_sync(&topology_timer);
  1342. }
  1343. return rc;
  1344. }
  1345. int prrn_is_enabled(void)
  1346. {
  1347. return prrn_enabled;
  1348. }
  1349. static int topology_read(struct seq_file *file, void *v)
  1350. {
  1351. if (vphn_enabled || prrn_enabled)
  1352. seq_puts(file, "on\n");
  1353. else
  1354. seq_puts(file, "off\n");
  1355. return 0;
  1356. }
  1357. static int topology_open(struct inode *inode, struct file *file)
  1358. {
  1359. return single_open(file, topology_read, NULL);
  1360. }
  1361. static ssize_t topology_write(struct file *file, const char __user *buf,
  1362. size_t count, loff_t *off)
  1363. {
  1364. char kbuf[4]; /* "on" or "off" plus null. */
  1365. int read_len;
  1366. read_len = count < 3 ? count : 3;
  1367. if (copy_from_user(kbuf, buf, read_len))
  1368. return -EINVAL;
  1369. kbuf[read_len] = '\0';
  1370. if (!strncmp(kbuf, "on", 2))
  1371. start_topology_update();
  1372. else if (!strncmp(kbuf, "off", 3))
  1373. stop_topology_update();
  1374. else
  1375. return -EINVAL;
  1376. return count;
  1377. }
  1378. static const struct file_operations topology_ops = {
  1379. .read = seq_read,
  1380. .write = topology_write,
  1381. .open = topology_open,
  1382. .release = single_release
  1383. };
  1384. static int topology_update_init(void)
  1385. {
  1386. start_topology_update();
  1387. proc_create("powerpc/topology_updates", 644, NULL, &topology_ops);
  1388. return 0;
  1389. }
  1390. device_initcall(topology_update_init);
  1391. #endif /* CONFIG_PPC_SPLPAR */