setup64.c 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337
  1. /*
  2. * X86-64 specific CPU setup.
  3. * Copyright (C) 1995 Linus Torvalds
  4. * Copyright 2001, 2002, 2003 SuSE Labs / Andi Kleen.
  5. * See setup.c for older changelog.
  6. */
  7. #include <linux/init.h>
  8. #include <linux/kernel.h>
  9. #include <linux/sched.h>
  10. #include <linux/string.h>
  11. #include <linux/bootmem.h>
  12. #include <linux/bitops.h>
  13. #include <linux/module.h>
  14. #include <asm/pda.h>
  15. #include <asm/pgtable.h>
  16. #include <asm/processor.h>
  17. #include <asm/desc.h>
  18. #include <asm/atomic.h>
  19. #include <asm/mmu_context.h>
  20. #include <asm/smp.h>
  21. #include <asm/i387.h>
  22. #include <asm/percpu.h>
  23. #include <asm/proto.h>
  24. #include <asm/sections.h>
  25. #include <asm/setup.h>
  26. #ifndef CONFIG_DEBUG_BOOT_PARAMS
  27. struct boot_params __initdata boot_params;
  28. #else
  29. struct boot_params boot_params;
  30. #endif
  31. cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE;
  32. struct x8664_pda *_cpu_pda[NR_CPUS] __read_mostly;
  33. EXPORT_SYMBOL(_cpu_pda);
  34. struct x8664_pda boot_cpu_pda[NR_CPUS] __cacheline_aligned;
  35. struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
  36. char boot_cpu_stack[IRQSTACKSIZE] __attribute__((section(".bss.page_aligned")));
  37. unsigned long __supported_pte_mask __read_mostly = ~0UL;
  38. static int do_not_nx __cpuinitdata = 0;
  39. /* noexec=on|off
  40. Control non executable mappings for 64bit processes.
  41. on Enable(default)
  42. off Disable
  43. */
  44. static int __init nonx_setup(char *str)
  45. {
  46. if (!str)
  47. return -EINVAL;
  48. if (!strncmp(str, "on", 2)) {
  49. __supported_pte_mask |= _PAGE_NX;
  50. do_not_nx = 0;
  51. } else if (!strncmp(str, "off", 3)) {
  52. do_not_nx = 1;
  53. __supported_pte_mask &= ~_PAGE_NX;
  54. }
  55. return 0;
  56. }
  57. early_param("noexec", nonx_setup);
  58. int force_personality32 = 0;
  59. /* noexec32=on|off
  60. Control non executable heap for 32bit processes.
  61. To control the stack too use noexec=off
  62. on PROT_READ does not imply PROT_EXEC for 32bit processes
  63. off PROT_READ implies PROT_EXEC (default)
  64. */
  65. static int __init nonx32_setup(char *str)
  66. {
  67. if (!strcmp(str, "on"))
  68. force_personality32 &= ~READ_IMPLIES_EXEC;
  69. else if (!strcmp(str, "off"))
  70. force_personality32 |= READ_IMPLIES_EXEC;
  71. return 1;
  72. }
  73. __setup("noexec32=", nonx32_setup);
  74. /*
  75. * Copy data used in early init routines from the initial arrays to the
  76. * per cpu data areas. These arrays then become expendable and the *_ptrs
  77. * are zeroed indicating that the static arrays are gone.
  78. */
  79. void __init setup_percpu_maps(void)
  80. {
  81. int cpu;
  82. for_each_possible_cpu(cpu) {
  83. #ifdef CONFIG_SMP
  84. if (per_cpu_offset(cpu)) {
  85. #endif
  86. per_cpu(x86_cpu_to_apicid, cpu) =
  87. x86_cpu_to_apicid_init[cpu];
  88. #ifdef CONFIG_NUMA
  89. per_cpu(x86_cpu_to_node_map, cpu) =
  90. x86_cpu_to_node_map_init[cpu];
  91. #endif
  92. #ifdef CONFIG_SMP
  93. }
  94. else
  95. printk(KERN_NOTICE "per_cpu_offset zero for cpu %d\n",
  96. cpu);
  97. #endif
  98. }
  99. /* indicate the early static arrays are gone */
  100. x86_cpu_to_apicid_early_ptr = NULL;
  101. #ifdef CONFIG_NUMA
  102. x86_cpu_to_node_map_early_ptr = NULL;
  103. #endif
  104. }
  105. /*
  106. * Great future plan:
  107. * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
  108. * Always point %gs to its beginning
  109. */
  110. void __init setup_per_cpu_areas(void)
  111. {
  112. int i;
  113. unsigned long size;
  114. #ifdef CONFIG_HOTPLUG_CPU
  115. prefill_possible_map();
  116. #endif
  117. /* Copy section for each CPU (we discard the original) */
  118. size = PERCPU_ENOUGH_ROOM;
  119. printk(KERN_INFO "PERCPU: Allocating %lu bytes of per cpu data\n", size);
  120. for_each_cpu_mask (i, cpu_possible_map) {
  121. char *ptr;
  122. if (!NODE_DATA(early_cpu_to_node(i))) {
  123. printk("cpu with no node %d, num_online_nodes %d\n",
  124. i, num_online_nodes());
  125. ptr = alloc_bootmem_pages(size);
  126. } else {
  127. ptr = alloc_bootmem_pages_node(NODE_DATA(early_cpu_to_node(i)), size);
  128. }
  129. if (!ptr)
  130. panic("Cannot allocate cpu data for CPU %d\n", i);
  131. cpu_pda(i)->data_offset = ptr - __per_cpu_start;
  132. memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
  133. }
  134. /* setup percpu data maps early */
  135. setup_percpu_maps();
  136. }
  137. void pda_init(int cpu)
  138. {
  139. struct x8664_pda *pda = cpu_pda(cpu);
  140. /* Setup up data that may be needed in __get_free_pages early */
  141. asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
  142. /* Memory clobbers used to order PDA accessed */
  143. mb();
  144. wrmsrl(MSR_GS_BASE, pda);
  145. mb();
  146. pda->cpunumber = cpu;
  147. pda->irqcount = -1;
  148. pda->kernelstack =
  149. (unsigned long)stack_thread_info() - PDA_STACKOFFSET + THREAD_SIZE;
  150. pda->active_mm = &init_mm;
  151. pda->mmu_state = 0;
  152. if (cpu == 0) {
  153. /* others are initialized in smpboot.c */
  154. pda->pcurrent = &init_task;
  155. pda->irqstackptr = boot_cpu_stack;
  156. } else {
  157. pda->irqstackptr = (char *)
  158. __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
  159. if (!pda->irqstackptr)
  160. panic("cannot allocate irqstack for cpu %d", cpu);
  161. }
  162. pda->irqstackptr += IRQSTACKSIZE-64;
  163. }
  164. char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]
  165. __attribute__((section(".bss.page_aligned")));
  166. extern asmlinkage void ignore_sysret(void);
  167. /* May not be marked __init: used by software suspend */
  168. void syscall_init(void)
  169. {
  170. /*
  171. * LSTAR and STAR live in a bit strange symbiosis.
  172. * They both write to the same internal register. STAR allows to set CS/DS
  173. * but only a 32bit target. LSTAR sets the 64bit rip.
  174. */
  175. wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 | ((u64)__KERNEL_CS)<<32);
  176. wrmsrl(MSR_LSTAR, system_call);
  177. wrmsrl(MSR_CSTAR, ignore_sysret);
  178. #ifdef CONFIG_IA32_EMULATION
  179. syscall32_cpu_init ();
  180. #endif
  181. /* Flags to clear on syscall */
  182. wrmsrl(MSR_SYSCALL_MASK,
  183. X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
  184. }
  185. void __cpuinit check_efer(void)
  186. {
  187. unsigned long efer;
  188. rdmsrl(MSR_EFER, efer);
  189. if (!(efer & EFER_NX) || do_not_nx) {
  190. __supported_pte_mask &= ~_PAGE_NX;
  191. }
  192. }
  193. unsigned long kernel_eflags;
  194. /*
  195. * Copies of the original ist values from the tss are only accessed during
  196. * debugging, no special alignment required.
  197. */
  198. DEFINE_PER_CPU(struct orig_ist, orig_ist);
  199. /*
  200. * cpu_init() initializes state that is per-CPU. Some data is already
  201. * initialized (naturally) in the bootstrap process, such as the GDT
  202. * and IDT. We reload them nevertheless, this function acts as a
  203. * 'CPU state barrier', nothing should get across.
  204. * A lot of state is already set up in PDA init.
  205. */
  206. void __cpuinit cpu_init (void)
  207. {
  208. int cpu = stack_smp_processor_id();
  209. struct tss_struct *t = &per_cpu(init_tss, cpu);
  210. struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
  211. unsigned long v;
  212. char *estacks = NULL;
  213. struct task_struct *me;
  214. int i;
  215. /* CPU 0 is initialised in head64.c */
  216. if (cpu != 0) {
  217. pda_init(cpu);
  218. } else
  219. estacks = boot_exception_stacks;
  220. me = current;
  221. if (cpu_test_and_set(cpu, cpu_initialized))
  222. panic("CPU#%d already initialized!\n", cpu);
  223. printk("Initializing CPU#%d\n", cpu);
  224. clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
  225. /*
  226. * Initialize the per-CPU GDT with the boot GDT,
  227. * and set up the GDT descriptor:
  228. */
  229. if (cpu)
  230. memcpy(get_cpu_gdt_table(cpu), cpu_gdt_table, GDT_SIZE);
  231. cpu_gdt_descr[cpu].size = GDT_SIZE;
  232. load_gdt((const struct desc_ptr *)&cpu_gdt_descr[cpu]);
  233. load_idt((const struct desc_ptr *)&idt_descr);
  234. memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
  235. syscall_init();
  236. wrmsrl(MSR_FS_BASE, 0);
  237. wrmsrl(MSR_KERNEL_GS_BASE, 0);
  238. barrier();
  239. check_efer();
  240. /*
  241. * set up and load the per-CPU TSS
  242. */
  243. for (v = 0; v < N_EXCEPTION_STACKS; v++) {
  244. static const unsigned int order[N_EXCEPTION_STACKS] = {
  245. [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
  246. [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
  247. };
  248. if (cpu) {
  249. estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
  250. if (!estacks)
  251. panic("Cannot allocate exception stack %ld %d\n",
  252. v, cpu);
  253. }
  254. estacks += PAGE_SIZE << order[v];
  255. orig_ist->ist[v] = t->x86_tss.ist[v] = (unsigned long)estacks;
  256. }
  257. t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
  258. /*
  259. * <= is required because the CPU will access up to
  260. * 8 bits beyond the end of the IO permission bitmap.
  261. */
  262. for (i = 0; i <= IO_BITMAP_LONGS; i++)
  263. t->io_bitmap[i] = ~0UL;
  264. atomic_inc(&init_mm.mm_count);
  265. me->active_mm = &init_mm;
  266. if (me->mm)
  267. BUG();
  268. enter_lazy_tlb(&init_mm, me);
  269. set_tss_desc(cpu, t);
  270. load_TR_desc();
  271. load_LDT(&init_mm.context);
  272. /*
  273. * Clear all 6 debug registers:
  274. */
  275. set_debugreg(0UL, 0);
  276. set_debugreg(0UL, 1);
  277. set_debugreg(0UL, 2);
  278. set_debugreg(0UL, 3);
  279. set_debugreg(0UL, 6);
  280. set_debugreg(0UL, 7);
  281. fpu_init();
  282. raw_local_save_flags(kernel_eflags);
  283. }