setup_percpu.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. #include <linux/kernel.h>
  2. #include <linux/module.h>
  3. #include <linux/init.h>
  4. #include <linux/bootmem.h>
  5. #include <linux/percpu.h>
  6. #include <linux/kexec.h>
  7. #include <linux/crash_dump.h>
  8. #include <linux/smp.h>
  9. #include <linux/topology.h>
  10. #include <asm/sections.h>
  11. #include <asm/processor.h>
  12. #include <asm/setup.h>
  13. #include <asm/mpspec.h>
  14. #include <asm/apicdef.h>
  15. #include <asm/highmem.h>
  16. #include <asm/proto.h>
  17. #include <asm/cpumask.h>
  18. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  19. # define DBG(x...) printk(KERN_DEBUG x)
  20. #else
  21. # define DBG(x...)
  22. #endif
  23. /*
  24. * Could be inside CONFIG_HAVE_SETUP_PER_CPU_AREA with other stuff but
  25. * voyager wants cpu_number too.
  26. */
  27. #ifdef CONFIG_SMP
  28. DEFINE_PER_CPU(int, cpu_number);
  29. EXPORT_PER_CPU_SYMBOL(cpu_number);
  30. #endif
  31. #ifdef CONFIG_X86_LOCAL_APIC
  32. unsigned int num_processors;
  33. unsigned disabled_cpus __cpuinitdata;
  34. /* Processor that is doing the boot up */
  35. unsigned int boot_cpu_physical_apicid = -1U;
  36. EXPORT_SYMBOL(boot_cpu_physical_apicid);
  37. unsigned int max_physical_apicid;
  38. /* Bitmask of physically existing CPUs */
  39. physid_mask_t phys_cpu_present_map;
  40. #endif
  41. /*
  42. * Map cpu index to physical APIC ID
  43. */
  44. DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
  45. DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
  46. EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
  47. EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  48. #if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
  49. #define X86_64_NUMA 1 /* (used later) */
  50. DEFINE_PER_CPU(int, node_number) = 0;
  51. EXPORT_PER_CPU_SYMBOL(node_number);
  52. /*
  53. * Map cpu index to node index
  54. */
  55. DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
  56. EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
  57. /*
  58. * Which logical CPUs are on which nodes
  59. */
  60. cpumask_t *node_to_cpumask_map;
  61. EXPORT_SYMBOL(node_to_cpumask_map);
  62. /*
  63. * Setup node_to_cpumask_map
  64. */
  65. static void __init setup_node_to_cpumask_map(void);
  66. #else
  67. static inline void setup_node_to_cpumask_map(void) { }
  68. #endif
  69. /*
  70. * Define load_pda_offset() and per-cpu __pda for x86_64.
  71. * load_pda_offset() is responsible for loading the offset of pda into
  72. * %gs.
  73. *
  74. * On SMP, pda offset also duals as percpu base address and thus it
  75. * should be at the start of per-cpu area. To achieve this, it's
  76. * preallocated in vmlinux_64.lds.S directly instead of using
  77. * DEFINE_PER_CPU().
  78. */
  79. #ifdef CONFIG_X86_64
  80. void __cpuinit load_pda_offset(int cpu)
  81. {
  82. /* Memory clobbers used to order pda/percpu accesses */
  83. mb();
  84. wrmsrl(MSR_GS_BASE, cpu_pda(cpu));
  85. mb();
  86. }
  87. #ifndef CONFIG_SMP
  88. DEFINE_PER_CPU(struct x8664_pda, __pda);
  89. #endif
  90. EXPORT_PER_CPU_SYMBOL(__pda);
  91. #endif /* CONFIG_SMP && CONFIG_X86_64 */
  92. #ifdef CONFIG_X86_64
  93. /* correctly size the local cpu masks */
  94. static void setup_cpu_local_masks(void)
  95. {
  96. alloc_bootmem_cpumask_var(&cpu_initialized_mask);
  97. alloc_bootmem_cpumask_var(&cpu_callin_mask);
  98. alloc_bootmem_cpumask_var(&cpu_callout_mask);
  99. alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
  100. }
  101. #else /* CONFIG_X86_32 */
  102. static inline void setup_cpu_local_masks(void)
  103. {
  104. }
  105. #endif /* CONFIG_X86_32 */
  106. #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  107. /*
  108. * Copy data used in early init routines from the initial arrays to the
  109. * per cpu data areas. These arrays then become expendable and the
  110. * *_early_ptr's are zeroed indicating that the static arrays are gone.
  111. */
  112. static void __init setup_per_cpu_maps(void)
  113. {
  114. int cpu;
  115. for_each_possible_cpu(cpu) {
  116. per_cpu(x86_cpu_to_apicid, cpu) =
  117. early_per_cpu_map(x86_cpu_to_apicid, cpu);
  118. per_cpu(x86_bios_cpu_apicid, cpu) =
  119. early_per_cpu_map(x86_bios_cpu_apicid, cpu);
  120. #ifdef X86_64_NUMA
  121. per_cpu(x86_cpu_to_node_map, cpu) =
  122. early_per_cpu_map(x86_cpu_to_node_map, cpu);
  123. #endif
  124. }
  125. /* indicate the early static arrays will soon be gone */
  126. early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
  127. early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
  128. #ifdef X86_64_NUMA
  129. early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
  130. #endif
  131. }
  132. #ifdef CONFIG_X86_64
  133. unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
  134. [0] = (unsigned long)__per_cpu_load,
  135. };
  136. #else
  137. unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
  138. #endif
  139. EXPORT_SYMBOL(__per_cpu_offset);
  140. /*
  141. * Great future plan:
  142. * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
  143. * Always point %gs to its beginning
  144. */
  145. void __init setup_per_cpu_areas(void)
  146. {
  147. ssize_t size, old_size;
  148. char *ptr;
  149. int cpu;
  150. unsigned long align = 1;
  151. /* Copy section for each CPU (we discard the original) */
  152. old_size = PERCPU_ENOUGH_ROOM;
  153. align = max_t(unsigned long, PAGE_SIZE, align);
  154. size = roundup(old_size, align);
  155. pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
  156. NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
  157. pr_info("PERCPU: Allocating %zd bytes of per cpu data\n", size);
  158. for_each_possible_cpu(cpu) {
  159. #ifndef CONFIG_NEED_MULTIPLE_NODES
  160. ptr = __alloc_bootmem(size, align,
  161. __pa(MAX_DMA_ADDRESS));
  162. #else
  163. int node = early_cpu_to_node(cpu);
  164. if (!node_online(node) || !NODE_DATA(node)) {
  165. ptr = __alloc_bootmem(size, align,
  166. __pa(MAX_DMA_ADDRESS));
  167. pr_info("cpu %d has no node %d or node-local memory\n",
  168. cpu, node);
  169. pr_debug("per cpu data for cpu%d at %016lx\n",
  170. cpu, __pa(ptr));
  171. } else {
  172. ptr = __alloc_bootmem_node(NODE_DATA(node), size, align,
  173. __pa(MAX_DMA_ADDRESS));
  174. pr_debug("per cpu data for cpu%d on node%d at %016lx\n",
  175. cpu, node, __pa(ptr));
  176. }
  177. #endif
  178. memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
  179. per_cpu_offset(cpu) = ptr - __per_cpu_start;
  180. per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
  181. per_cpu(cpu_number, cpu) = cpu;
  182. #ifdef CONFIG_X86_64
  183. per_cpu(irq_stack_ptr, cpu) =
  184. (char *)per_cpu(irq_stack, cpu) + IRQ_STACK_SIZE - 64;
  185. /*
  186. * CPU0 modified pda in the init data area, reload pda
  187. * offset for CPU0 and clear the area for others.
  188. */
  189. if (cpu == 0)
  190. load_pda_offset(0);
  191. else
  192. memset(cpu_pda(cpu), 0, sizeof(*cpu_pda(cpu)));
  193. #endif
  194. DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
  195. }
  196. /* Setup percpu data maps */
  197. setup_per_cpu_maps();
  198. /* Setup node to cpumask map */
  199. setup_node_to_cpumask_map();
  200. /* Setup cpu initialized, callin, callout masks */
  201. setup_cpu_local_masks();
  202. }
  203. #endif
  204. #ifdef X86_64_NUMA
  205. /*
  206. * Allocate node_to_cpumask_map based on number of available nodes
  207. * Requires node_possible_map to be valid.
  208. *
  209. * Note: node_to_cpumask() is not valid until after this is done.
  210. * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
  211. */
  212. static void __init setup_node_to_cpumask_map(void)
  213. {
  214. unsigned int node, num = 0;
  215. cpumask_t *map;
  216. /* setup nr_node_ids if not done yet */
  217. if (nr_node_ids == MAX_NUMNODES) {
  218. for_each_node_mask(node, node_possible_map)
  219. num = node;
  220. nr_node_ids = num + 1;
  221. }
  222. /* allocate the map */
  223. map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
  224. DBG("node_to_cpumask_map at %p for %d nodes\n", map, nr_node_ids);
  225. pr_debug("Node to cpumask map at %p for %d nodes\n",
  226. map, nr_node_ids);
  227. /* node_to_cpumask() will now work */
  228. node_to_cpumask_map = map;
  229. }
  230. void __cpuinit numa_set_node(int cpu, int node)
  231. {
  232. int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
  233. /* early setting, no percpu area yet */
  234. if (cpu_to_node_map) {
  235. cpu_to_node_map[cpu] = node;
  236. return;
  237. }
  238. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  239. if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
  240. printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
  241. dump_stack();
  242. return;
  243. }
  244. #endif
  245. per_cpu(x86_cpu_to_node_map, cpu) = node;
  246. if (node != NUMA_NO_NODE)
  247. per_cpu(node_number, cpu) = node;
  248. }
  249. void __cpuinit numa_clear_node(int cpu)
  250. {
  251. numa_set_node(cpu, NUMA_NO_NODE);
  252. }
  253. #ifndef CONFIG_DEBUG_PER_CPU_MAPS
  254. void __cpuinit numa_add_cpu(int cpu)
  255. {
  256. cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
  257. }
  258. void __cpuinit numa_remove_cpu(int cpu)
  259. {
  260. cpu_clear(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
  261. }
  262. #else /* CONFIG_DEBUG_PER_CPU_MAPS */
  263. /*
  264. * --------- debug versions of the numa functions ---------
  265. */
  266. static void __cpuinit numa_set_cpumask(int cpu, int enable)
  267. {
  268. int node = early_cpu_to_node(cpu);
  269. cpumask_t *mask;
  270. char buf[64];
  271. if (node_to_cpumask_map == NULL) {
  272. printk(KERN_ERR "node_to_cpumask_map NULL\n");
  273. dump_stack();
  274. return;
  275. }
  276. mask = &node_to_cpumask_map[node];
  277. if (enable)
  278. cpu_set(cpu, *mask);
  279. else
  280. cpu_clear(cpu, *mask);
  281. cpulist_scnprintf(buf, sizeof(buf), mask);
  282. printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
  283. enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
  284. }
  285. void __cpuinit numa_add_cpu(int cpu)
  286. {
  287. numa_set_cpumask(cpu, 1);
  288. }
  289. void __cpuinit numa_remove_cpu(int cpu)
  290. {
  291. numa_set_cpumask(cpu, 0);
  292. }
  293. int cpu_to_node(int cpu)
  294. {
  295. if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
  296. printk(KERN_WARNING
  297. "cpu_to_node(%d): usage too early!\n", cpu);
  298. dump_stack();
  299. return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
  300. }
  301. return per_cpu(x86_cpu_to_node_map, cpu);
  302. }
  303. EXPORT_SYMBOL(cpu_to_node);
  304. /*
  305. * Same function as cpu_to_node() but used if called before the
  306. * per_cpu areas are setup.
  307. */
  308. int early_cpu_to_node(int cpu)
  309. {
  310. if (early_per_cpu_ptr(x86_cpu_to_node_map))
  311. return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
  312. if (!per_cpu_offset(cpu)) {
  313. printk(KERN_WARNING
  314. "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
  315. dump_stack();
  316. return NUMA_NO_NODE;
  317. }
  318. return per_cpu(x86_cpu_to_node_map, cpu);
  319. }
  320. /* empty cpumask */
  321. static const cpumask_t cpu_mask_none;
  322. /*
  323. * Returns a pointer to the bitmask of CPUs on Node 'node'.
  324. */
  325. const cpumask_t *cpumask_of_node(int node)
  326. {
  327. if (node_to_cpumask_map == NULL) {
  328. printk(KERN_WARNING
  329. "cpumask_of_node(%d): no node_to_cpumask_map!\n",
  330. node);
  331. dump_stack();
  332. return (const cpumask_t *)&cpu_online_map;
  333. }
  334. if (node >= nr_node_ids) {
  335. printk(KERN_WARNING
  336. "cpumask_of_node(%d): node > nr_node_ids(%d)\n",
  337. node, nr_node_ids);
  338. dump_stack();
  339. return &cpu_mask_none;
  340. }
  341. return &node_to_cpumask_map[node];
  342. }
  343. EXPORT_SYMBOL(cpumask_of_node);
  344. /*
  345. * Returns a bitmask of CPUs on Node 'node'.
  346. *
  347. * Side note: this function creates the returned cpumask on the stack
  348. * so with a high NR_CPUS count, excessive stack space is used. The
  349. * node_to_cpumask_ptr function should be used whenever possible.
  350. */
  351. cpumask_t node_to_cpumask(int node)
  352. {
  353. if (node_to_cpumask_map == NULL) {
  354. printk(KERN_WARNING
  355. "node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
  356. dump_stack();
  357. return cpu_online_map;
  358. }
  359. if (node >= nr_node_ids) {
  360. printk(KERN_WARNING
  361. "node_to_cpumask(%d): node > nr_node_ids(%d)\n",
  362. node, nr_node_ids);
  363. dump_stack();
  364. return cpu_mask_none;
  365. }
  366. return node_to_cpumask_map[node];
  367. }
  368. EXPORT_SYMBOL(node_to_cpumask);
  369. /*
  370. * --------- end of debug versions of the numa functions ---------
  371. */
  372. #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
  373. #endif /* X86_64_NUMA */