common.c 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380
  1. #include <linux/bootmem.h>
  2. #include <linux/linkage.h>
  3. #include <linux/bitops.h>
  4. #include <linux/kernel.h>
  5. #include <linux/module.h>
  6. #include <linux/percpu.h>
  7. #include <linux/string.h>
  8. #include <linux/delay.h>
  9. #include <linux/sched.h>
  10. #include <linux/init.h>
  11. #include <linux/kgdb.h>
  12. #include <linux/smp.h>
  13. #include <linux/io.h>
  14. #include <asm/stackprotector.h>
  15. #include <asm/perf_event.h>
  16. #include <asm/mmu_context.h>
  17. #include <asm/archrandom.h>
  18. #include <asm/hypervisor.h>
  19. #include <asm/processor.h>
  20. #include <asm/debugreg.h>
  21. #include <asm/sections.h>
  22. #include <linux/topology.h>
  23. #include <linux/cpumask.h>
  24. #include <asm/pgtable.h>
  25. #include <linux/atomic.h>
  26. #include <asm/proto.h>
  27. #include <asm/setup.h>
  28. #include <asm/apic.h>
  29. #include <asm/desc.h>
  30. #include <asm/i387.h>
  31. #include <asm/fpu-internal.h>
  32. #include <asm/mtrr.h>
  33. #include <linux/numa.h>
  34. #include <asm/asm.h>
  35. #include <asm/cpu.h>
  36. #include <asm/mce.h>
  37. #include <asm/msr.h>
  38. #include <asm/pat.h>
  39. #include <asm/microcode.h>
  40. #include <asm/microcode_intel.h>
  41. #ifdef CONFIG_X86_LOCAL_APIC
  42. #include <asm/uv/uv.h>
  43. #endif
  44. #include "cpu.h"
  45. /* all of these masks are initialized in setup_cpu_local_masks() */
  46. cpumask_var_t cpu_initialized_mask;
  47. cpumask_var_t cpu_callout_mask;
  48. cpumask_var_t cpu_callin_mask;
  49. /* representing cpus for which sibling maps can be computed */
  50. cpumask_var_t cpu_sibling_setup_mask;
  51. /* correctly size the local cpu masks */
  52. void __init setup_cpu_local_masks(void)
  53. {
  54. alloc_bootmem_cpumask_var(&cpu_initialized_mask);
  55. alloc_bootmem_cpumask_var(&cpu_callin_mask);
  56. alloc_bootmem_cpumask_var(&cpu_callout_mask);
  57. alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
  58. }
  59. static void default_init(struct cpuinfo_x86 *c)
  60. {
  61. #ifdef CONFIG_X86_64
  62. cpu_detect_cache_sizes(c);
  63. #else
  64. /* Not much we can do here... */
  65. /* Check if at least it has cpuid */
  66. if (c->cpuid_level == -1) {
  67. /* No cpuid. It must be an ancient CPU */
  68. if (c->x86 == 4)
  69. strcpy(c->x86_model_id, "486");
  70. else if (c->x86 == 3)
  71. strcpy(c->x86_model_id, "386");
  72. }
  73. #endif
  74. }
  75. static const struct cpu_dev default_cpu = {
  76. .c_init = default_init,
  77. .c_vendor = "Unknown",
  78. .c_x86_vendor = X86_VENDOR_UNKNOWN,
  79. };
  80. static const struct cpu_dev *this_cpu = &default_cpu;
  81. DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
  82. #ifdef CONFIG_X86_64
  83. /*
  84. * We need valid kernel segments for data and code in long mode too
  85. * IRET will check the segment types kkeil 2000/10/28
  86. * Also sysret mandates a special GDT layout
  87. *
  88. * TLS descriptors are currently at a different place compared to i386.
  89. * Hopefully nobody expects them at a fixed place (Wine?)
  90. */
  91. [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
  92. [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
  93. [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
  94. [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
  95. [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
  96. [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
  97. #else
  98. [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
  99. [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
  100. [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
  101. [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
  102. /*
  103. * Segments used for calling PnP BIOS have byte granularity.
  104. * They code segments and data segments have fixed 64k limits,
  105. * the transfer segment sizes are set at run time.
  106. */
  107. /* 32-bit code */
  108. [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
  109. /* 16-bit code */
  110. [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
  111. /* 16-bit data */
  112. [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
  113. /* 16-bit data */
  114. [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(0x0092, 0, 0),
  115. /* 16-bit data */
  116. [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(0x0092, 0, 0),
  117. /*
  118. * The APM segments have byte granularity and their bases
  119. * are set at run time. All have 64k limits.
  120. */
  121. /* 32-bit code */
  122. [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
  123. /* 16-bit code */
  124. [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
  125. /* data */
  126. [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
  127. [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
  128. [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
  129. GDT_STACK_CANARY_INIT
  130. #endif
  131. } };
  132. EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
  133. static int __init x86_xsave_setup(char *s)
  134. {
  135. setup_clear_cpu_cap(X86_FEATURE_XSAVE);
  136. setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
  137. setup_clear_cpu_cap(X86_FEATURE_AVX);
  138. setup_clear_cpu_cap(X86_FEATURE_AVX2);
  139. return 1;
  140. }
  141. __setup("noxsave", x86_xsave_setup);
  142. static int __init x86_xsaveopt_setup(char *s)
  143. {
  144. setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
  145. return 1;
  146. }
  147. __setup("noxsaveopt", x86_xsaveopt_setup);
  148. #ifdef CONFIG_X86_32
  149. static int cachesize_override = -1;
  150. static int disable_x86_serial_nr = 1;
  151. static int __init cachesize_setup(char *str)
  152. {
  153. get_option(&str, &cachesize_override);
  154. return 1;
  155. }
  156. __setup("cachesize=", cachesize_setup);
  157. static int __init x86_fxsr_setup(char *s)
  158. {
  159. setup_clear_cpu_cap(X86_FEATURE_FXSR);
  160. setup_clear_cpu_cap(X86_FEATURE_XMM);
  161. return 1;
  162. }
  163. __setup("nofxsr", x86_fxsr_setup);
  164. static int __init x86_sep_setup(char *s)
  165. {
  166. setup_clear_cpu_cap(X86_FEATURE_SEP);
  167. return 1;
  168. }
  169. __setup("nosep", x86_sep_setup);
  170. /* Standard macro to see if a specific flag is changeable */
  171. static inline int flag_is_changeable_p(u32 flag)
  172. {
  173. u32 f1, f2;
  174. /*
  175. * Cyrix and IDT cpus allow disabling of CPUID
  176. * so the code below may return different results
  177. * when it is executed before and after enabling
  178. * the CPUID. Add "volatile" to not allow gcc to
  179. * optimize the subsequent calls to this function.
  180. */
  181. asm volatile ("pushfl \n\t"
  182. "pushfl \n\t"
  183. "popl %0 \n\t"
  184. "movl %0, %1 \n\t"
  185. "xorl %2, %0 \n\t"
  186. "pushl %0 \n\t"
  187. "popfl \n\t"
  188. "pushfl \n\t"
  189. "popl %0 \n\t"
  190. "popfl \n\t"
  191. : "=&r" (f1), "=&r" (f2)
  192. : "ir" (flag));
  193. return ((f1^f2) & flag) != 0;
  194. }
  195. /* Probe for the CPUID instruction */
  196. int have_cpuid_p(void)
  197. {
  198. return flag_is_changeable_p(X86_EFLAGS_ID);
  199. }
  200. static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
  201. {
  202. unsigned long lo, hi;
  203. if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
  204. return;
  205. /* Disable processor serial number: */
  206. rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
  207. lo |= 0x200000;
  208. wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
  209. printk(KERN_NOTICE "CPU serial number disabled.\n");
  210. clear_cpu_cap(c, X86_FEATURE_PN);
  211. /* Disabling the serial number may affect the cpuid level */
  212. c->cpuid_level = cpuid_eax(0);
  213. }
  214. static int __init x86_serial_nr_setup(char *s)
  215. {
  216. disable_x86_serial_nr = 0;
  217. return 1;
  218. }
  219. __setup("serialnumber", x86_serial_nr_setup);
  220. #else
  221. static inline int flag_is_changeable_p(u32 flag)
  222. {
  223. return 1;
  224. }
  225. static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
  226. {
  227. }
  228. #endif
  229. static __init int setup_disable_smep(char *arg)
  230. {
  231. setup_clear_cpu_cap(X86_FEATURE_SMEP);
  232. return 1;
  233. }
  234. __setup("nosmep", setup_disable_smep);
  235. static __always_inline void setup_smep(struct cpuinfo_x86 *c)
  236. {
  237. if (cpu_has(c, X86_FEATURE_SMEP))
  238. set_in_cr4(X86_CR4_SMEP);
  239. }
  240. static __init int setup_disable_smap(char *arg)
  241. {
  242. setup_clear_cpu_cap(X86_FEATURE_SMAP);
  243. return 1;
  244. }
  245. __setup("nosmap", setup_disable_smap);
  246. static __always_inline void setup_smap(struct cpuinfo_x86 *c)
  247. {
  248. unsigned long eflags;
  249. /* This should have been cleared long ago */
  250. raw_local_save_flags(eflags);
  251. BUG_ON(eflags & X86_EFLAGS_AC);
  252. if (cpu_has(c, X86_FEATURE_SMAP))
  253. set_in_cr4(X86_CR4_SMAP);
  254. }
  255. /*
  256. * Some CPU features depend on higher CPUID levels, which may not always
  257. * be available due to CPUID level capping or broken virtualization
  258. * software. Add those features to this table to auto-disable them.
  259. */
  260. struct cpuid_dependent_feature {
  261. u32 feature;
  262. u32 level;
  263. };
  264. static const struct cpuid_dependent_feature
  265. cpuid_dependent_features[] = {
  266. { X86_FEATURE_MWAIT, 0x00000005 },
  267. { X86_FEATURE_DCA, 0x00000009 },
  268. { X86_FEATURE_XSAVE, 0x0000000d },
  269. { 0, 0 }
  270. };
  271. static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
  272. {
  273. const struct cpuid_dependent_feature *df;
  274. for (df = cpuid_dependent_features; df->feature; df++) {
  275. if (!cpu_has(c, df->feature))
  276. continue;
  277. /*
  278. * Note: cpuid_level is set to -1 if unavailable, but
  279. * extended_extended_level is set to 0 if unavailable
  280. * and the legitimate extended levels are all negative
  281. * when signed; hence the weird messing around with
  282. * signs here...
  283. */
  284. if (!((s32)df->level < 0 ?
  285. (u32)df->level > (u32)c->extended_cpuid_level :
  286. (s32)df->level > (s32)c->cpuid_level))
  287. continue;
  288. clear_cpu_cap(c, df->feature);
  289. if (!warn)
  290. continue;
  291. printk(KERN_WARNING
  292. "CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
  293. x86_cap_flags[df->feature], df->level);
  294. }
  295. }
  296. /*
  297. * Naming convention should be: <Name> [(<Codename>)]
  298. * This table only is used unless init_<vendor>() below doesn't set it;
  299. * in particular, if CPUID levels 0x80000002..4 are supported, this
  300. * isn't used
  301. */
  302. /* Look up CPU names by table lookup. */
  303. static const char *table_lookup_model(struct cpuinfo_x86 *c)
  304. {
  305. const struct cpu_model_info *info;
  306. if (c->x86_model >= 16)
  307. return NULL; /* Range check */
  308. if (!this_cpu)
  309. return NULL;
  310. info = this_cpu->c_models;
  311. while (info && info->family) {
  312. if (info->family == c->x86)
  313. return info->model_names[c->x86_model];
  314. info++;
  315. }
  316. return NULL; /* Not found */
  317. }
  318. __u32 cpu_caps_cleared[NCAPINTS];
  319. __u32 cpu_caps_set[NCAPINTS];
  320. void load_percpu_segment(int cpu)
  321. {
  322. #ifdef CONFIG_X86_32
  323. loadsegment(fs, __KERNEL_PERCPU);
  324. #else
  325. loadsegment(gs, 0);
  326. wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
  327. #endif
  328. load_stack_canary_segment();
  329. }
  330. /*
  331. * Current gdt points %fs at the "master" per-cpu area: after this,
  332. * it's on the real one.
  333. */
  334. void switch_to_new_gdt(int cpu)
  335. {
  336. struct desc_ptr gdt_descr;
  337. gdt_descr.address = (long)get_cpu_gdt_table(cpu);
  338. gdt_descr.size = GDT_SIZE - 1;
  339. load_gdt(&gdt_descr);
  340. /* Reload the per-cpu base */
  341. load_percpu_segment(cpu);
  342. }
  343. static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
  344. static void get_model_name(struct cpuinfo_x86 *c)
  345. {
  346. unsigned int *v;
  347. char *p, *q;
  348. if (c->extended_cpuid_level < 0x80000004)
  349. return;
  350. v = (unsigned int *)c->x86_model_id;
  351. cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
  352. cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
  353. cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
  354. c->x86_model_id[48] = 0;
  355. /*
  356. * Intel chips right-justify this string for some dumb reason;
  357. * undo that brain damage:
  358. */
  359. p = q = &c->x86_model_id[0];
  360. while (*p == ' ')
  361. p++;
  362. if (p != q) {
  363. while (*p)
  364. *q++ = *p++;
  365. while (q <= &c->x86_model_id[48])
  366. *q++ = '\0'; /* Zero-pad the rest */
  367. }
  368. }
  369. void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
  370. {
  371. unsigned int n, dummy, ebx, ecx, edx, l2size;
  372. n = c->extended_cpuid_level;
  373. if (n >= 0x80000005) {
  374. cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
  375. c->x86_cache_size = (ecx>>24) + (edx>>24);
  376. #ifdef CONFIG_X86_64
  377. /* On K8 L1 TLB is inclusive, so don't count it */
  378. c->x86_tlbsize = 0;
  379. #endif
  380. }
  381. if (n < 0x80000006) /* Some chips just has a large L1. */
  382. return;
  383. cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
  384. l2size = ecx >> 16;
  385. #ifdef CONFIG_X86_64
  386. c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
  387. #else
  388. /* do processor-specific cache resizing */
  389. if (this_cpu->c_size_cache)
  390. l2size = this_cpu->c_size_cache(c, l2size);
  391. /* Allow user to override all this if necessary. */
  392. if (cachesize_override != -1)
  393. l2size = cachesize_override;
  394. if (l2size == 0)
  395. return; /* Again, no L2 cache is possible */
  396. #endif
  397. c->x86_cache_size = l2size;
  398. }
  399. u16 __read_mostly tlb_lli_4k[NR_INFO];
  400. u16 __read_mostly tlb_lli_2m[NR_INFO];
  401. u16 __read_mostly tlb_lli_4m[NR_INFO];
  402. u16 __read_mostly tlb_lld_4k[NR_INFO];
  403. u16 __read_mostly tlb_lld_2m[NR_INFO];
  404. u16 __read_mostly tlb_lld_4m[NR_INFO];
  405. /*
  406. * tlb_flushall_shift shows the balance point in replacing cr3 write
  407. * with multiple 'invlpg'. It will do this replacement when
  408. * flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
  409. * If tlb_flushall_shift is -1, means the replacement will be disabled.
  410. */
  411. s8 __read_mostly tlb_flushall_shift = -1;
  412. void cpu_detect_tlb(struct cpuinfo_x86 *c)
  413. {
  414. if (this_cpu->c_detect_tlb)
  415. this_cpu->c_detect_tlb(c);
  416. printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
  417. "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
  418. "tlb_flushall_shift: %d\n",
  419. tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
  420. tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
  421. tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
  422. tlb_flushall_shift);
  423. }
  424. void detect_ht(struct cpuinfo_x86 *c)
  425. {
  426. #ifdef CONFIG_X86_HT
  427. u32 eax, ebx, ecx, edx;
  428. int index_msb, core_bits;
  429. static bool printed;
  430. if (!cpu_has(c, X86_FEATURE_HT))
  431. return;
  432. if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
  433. goto out;
  434. if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
  435. return;
  436. cpuid(1, &eax, &ebx, &ecx, &edx);
  437. smp_num_siblings = (ebx & 0xff0000) >> 16;
  438. if (smp_num_siblings == 1) {
  439. printk_once(KERN_INFO "CPU0: Hyper-Threading is disabled\n");
  440. goto out;
  441. }
  442. if (smp_num_siblings <= 1)
  443. goto out;
  444. index_msb = get_count_order(smp_num_siblings);
  445. c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
  446. smp_num_siblings = smp_num_siblings / c->x86_max_cores;
  447. index_msb = get_count_order(smp_num_siblings);
  448. core_bits = get_count_order(c->x86_max_cores);
  449. c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
  450. ((1 << core_bits) - 1);
  451. out:
  452. if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
  453. printk(KERN_INFO "CPU: Physical Processor ID: %d\n",
  454. c->phys_proc_id);
  455. printk(KERN_INFO "CPU: Processor Core ID: %d\n",
  456. c->cpu_core_id);
  457. printed = 1;
  458. }
  459. #endif
  460. }
  461. static void get_cpu_vendor(struct cpuinfo_x86 *c)
  462. {
  463. char *v = c->x86_vendor_id;
  464. int i;
  465. for (i = 0; i < X86_VENDOR_NUM; i++) {
  466. if (!cpu_devs[i])
  467. break;
  468. if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
  469. (cpu_devs[i]->c_ident[1] &&
  470. !strcmp(v, cpu_devs[i]->c_ident[1]))) {
  471. this_cpu = cpu_devs[i];
  472. c->x86_vendor = this_cpu->c_x86_vendor;
  473. return;
  474. }
  475. }
  476. printk_once(KERN_ERR
  477. "CPU: vendor_id '%s' unknown, using generic init.\n" \
  478. "CPU: Your system may be unstable.\n", v);
  479. c->x86_vendor = X86_VENDOR_UNKNOWN;
  480. this_cpu = &default_cpu;
  481. }
  482. void cpu_detect(struct cpuinfo_x86 *c)
  483. {
  484. /* Get vendor name */
  485. cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
  486. (unsigned int *)&c->x86_vendor_id[0],
  487. (unsigned int *)&c->x86_vendor_id[8],
  488. (unsigned int *)&c->x86_vendor_id[4]);
  489. c->x86 = 4;
  490. /* Intel-defined flags: level 0x00000001 */
  491. if (c->cpuid_level >= 0x00000001) {
  492. u32 junk, tfms, cap0, misc;
  493. cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
  494. c->x86 = (tfms >> 8) & 0xf;
  495. c->x86_model = (tfms >> 4) & 0xf;
  496. c->x86_mask = tfms & 0xf;
  497. if (c->x86 == 0xf)
  498. c->x86 += (tfms >> 20) & 0xff;
  499. if (c->x86 >= 0x6)
  500. c->x86_model += ((tfms >> 16) & 0xf) << 4;
  501. if (cap0 & (1<<19)) {
  502. c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
  503. c->x86_cache_alignment = c->x86_clflush_size;
  504. }
  505. }
  506. }
  507. void get_cpu_cap(struct cpuinfo_x86 *c)
  508. {
  509. u32 tfms, xlvl;
  510. u32 ebx;
  511. /* Intel-defined flags: level 0x00000001 */
  512. if (c->cpuid_level >= 0x00000001) {
  513. u32 capability, excap;
  514. cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
  515. c->x86_capability[0] = capability;
  516. c->x86_capability[4] = excap;
  517. }
  518. /* Additional Intel-defined flags: level 0x00000007 */
  519. if (c->cpuid_level >= 0x00000007) {
  520. u32 eax, ebx, ecx, edx;
  521. cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
  522. c->x86_capability[9] = ebx;
  523. }
  524. /* AMD-defined flags: level 0x80000001 */
  525. xlvl = cpuid_eax(0x80000000);
  526. c->extended_cpuid_level = xlvl;
  527. if ((xlvl & 0xffff0000) == 0x80000000) {
  528. if (xlvl >= 0x80000001) {
  529. c->x86_capability[1] = cpuid_edx(0x80000001);
  530. c->x86_capability[6] = cpuid_ecx(0x80000001);
  531. }
  532. }
  533. if (c->extended_cpuid_level >= 0x80000008) {
  534. u32 eax = cpuid_eax(0x80000008);
  535. c->x86_virt_bits = (eax >> 8) & 0xff;
  536. c->x86_phys_bits = eax & 0xff;
  537. }
  538. #ifdef CONFIG_X86_32
  539. else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
  540. c->x86_phys_bits = 36;
  541. #endif
  542. if (c->extended_cpuid_level >= 0x80000007)
  543. c->x86_power = cpuid_edx(0x80000007);
  544. init_scattered_cpuid_features(c);
  545. }
  546. static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
  547. {
  548. #ifdef CONFIG_X86_32
  549. int i;
  550. /*
  551. * First of all, decide if this is a 486 or higher
  552. * It's a 486 if we can modify the AC flag
  553. */
  554. if (flag_is_changeable_p(X86_EFLAGS_AC))
  555. c->x86 = 4;
  556. else
  557. c->x86 = 3;
  558. for (i = 0; i < X86_VENDOR_NUM; i++)
  559. if (cpu_devs[i] && cpu_devs[i]->c_identify) {
  560. c->x86_vendor_id[0] = 0;
  561. cpu_devs[i]->c_identify(c);
  562. if (c->x86_vendor_id[0]) {
  563. get_cpu_vendor(c);
  564. break;
  565. }
  566. }
  567. #endif
  568. }
  569. /*
  570. * Do minimum CPU detection early.
  571. * Fields really needed: vendor, cpuid_level, family, model, mask,
  572. * cache alignment.
  573. * The others are not touched to avoid unwanted side effects.
  574. *
  575. * WARNING: this function is only called on the BP. Don't add code here
  576. * that is supposed to run on all CPUs.
  577. */
  578. static void __init early_identify_cpu(struct cpuinfo_x86 *c)
  579. {
  580. #ifdef CONFIG_X86_64
  581. c->x86_clflush_size = 64;
  582. c->x86_phys_bits = 36;
  583. c->x86_virt_bits = 48;
  584. #else
  585. c->x86_clflush_size = 32;
  586. c->x86_phys_bits = 32;
  587. c->x86_virt_bits = 32;
  588. #endif
  589. c->x86_cache_alignment = c->x86_clflush_size;
  590. memset(&c->x86_capability, 0, sizeof c->x86_capability);
  591. c->extended_cpuid_level = 0;
  592. if (!have_cpuid_p())
  593. identify_cpu_without_cpuid(c);
  594. /* cyrix could have cpuid enabled via c_identify()*/
  595. if (!have_cpuid_p())
  596. return;
  597. cpu_detect(c);
  598. get_cpu_vendor(c);
  599. get_cpu_cap(c);
  600. fpu_detect(c);
  601. if (this_cpu->c_early_init)
  602. this_cpu->c_early_init(c);
  603. c->cpu_index = 0;
  604. filter_cpuid_features(c, false);
  605. if (this_cpu->c_bsp_init)
  606. this_cpu->c_bsp_init(c);
  607. setup_force_cpu_cap(X86_FEATURE_ALWAYS);
  608. }
  609. void __init early_cpu_init(void)
  610. {
  611. const struct cpu_dev *const *cdev;
  612. int count = 0;
  613. #ifdef CONFIG_PROCESSOR_SELECT
  614. printk(KERN_INFO "KERNEL supported cpus:\n");
  615. #endif
  616. for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
  617. const struct cpu_dev *cpudev = *cdev;
  618. if (count >= X86_VENDOR_NUM)
  619. break;
  620. cpu_devs[count] = cpudev;
  621. count++;
  622. #ifdef CONFIG_PROCESSOR_SELECT
  623. {
  624. unsigned int j;
  625. for (j = 0; j < 2; j++) {
  626. if (!cpudev->c_ident[j])
  627. continue;
  628. printk(KERN_INFO " %s %s\n", cpudev->c_vendor,
  629. cpudev->c_ident[j]);
  630. }
  631. }
  632. #endif
  633. }
  634. early_identify_cpu(&boot_cpu_data);
  635. }
  636. /*
  637. * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
  638. * unfortunately, that's not true in practice because of early VIA
  639. * chips and (more importantly) broken virtualizers that are not easy
  640. * to detect. In the latter case it doesn't even *fail* reliably, so
  641. * probing for it doesn't even work. Disable it completely on 32-bit
  642. * unless we can find a reliable way to detect all the broken cases.
  643. * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
  644. */
  645. static void detect_nopl(struct cpuinfo_x86 *c)
  646. {
  647. #ifdef CONFIG_X86_32
  648. clear_cpu_cap(c, X86_FEATURE_NOPL);
  649. #else
  650. set_cpu_cap(c, X86_FEATURE_NOPL);
  651. #endif
  652. }
  653. static void generic_identify(struct cpuinfo_x86 *c)
  654. {
  655. c->extended_cpuid_level = 0;
  656. if (!have_cpuid_p())
  657. identify_cpu_without_cpuid(c);
  658. /* cyrix could have cpuid enabled via c_identify()*/
  659. if (!have_cpuid_p())
  660. return;
  661. cpu_detect(c);
  662. get_cpu_vendor(c);
  663. get_cpu_cap(c);
  664. if (c->cpuid_level >= 0x00000001) {
  665. c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
  666. #ifdef CONFIG_X86_32
  667. # ifdef CONFIG_X86_HT
  668. c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
  669. # else
  670. c->apicid = c->initial_apicid;
  671. # endif
  672. #endif
  673. c->phys_proc_id = c->initial_apicid;
  674. }
  675. get_model_name(c); /* Default name */
  676. detect_nopl(c);
  677. }
  678. /*
  679. * This does the hard work of actually picking apart the CPU stuff...
  680. */
  681. static void identify_cpu(struct cpuinfo_x86 *c)
  682. {
  683. int i;
  684. c->loops_per_jiffy = loops_per_jiffy;
  685. c->x86_cache_size = -1;
  686. c->x86_vendor = X86_VENDOR_UNKNOWN;
  687. c->x86_model = c->x86_mask = 0; /* So far unknown... */
  688. c->x86_vendor_id[0] = '\0'; /* Unset */
  689. c->x86_model_id[0] = '\0'; /* Unset */
  690. c->x86_max_cores = 1;
  691. c->x86_coreid_bits = 0;
  692. #ifdef CONFIG_X86_64
  693. c->x86_clflush_size = 64;
  694. c->x86_phys_bits = 36;
  695. c->x86_virt_bits = 48;
  696. #else
  697. c->cpuid_level = -1; /* CPUID not detected */
  698. c->x86_clflush_size = 32;
  699. c->x86_phys_bits = 32;
  700. c->x86_virt_bits = 32;
  701. #endif
  702. c->x86_cache_alignment = c->x86_clflush_size;
  703. memset(&c->x86_capability, 0, sizeof c->x86_capability);
  704. generic_identify(c);
  705. if (this_cpu->c_identify)
  706. this_cpu->c_identify(c);
  707. /* Clear/Set all flags overriden by options, after probe */
  708. for (i = 0; i < NCAPINTS; i++) {
  709. c->x86_capability[i] &= ~cpu_caps_cleared[i];
  710. c->x86_capability[i] |= cpu_caps_set[i];
  711. }
  712. #ifdef CONFIG_X86_64
  713. c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
  714. #endif
  715. /*
  716. * Vendor-specific initialization. In this section we
  717. * canonicalize the feature flags, meaning if there are
  718. * features a certain CPU supports which CPUID doesn't
  719. * tell us, CPUID claiming incorrect flags, or other bugs,
  720. * we handle them here.
  721. *
  722. * At the end of this section, c->x86_capability better
  723. * indicate the features this CPU genuinely supports!
  724. */
  725. if (this_cpu->c_init)
  726. this_cpu->c_init(c);
  727. /* Disable the PN if appropriate */
  728. squash_the_stupid_serial_number(c);
  729. /* Set up SMEP/SMAP */
  730. setup_smep(c);
  731. setup_smap(c);
  732. /*
  733. * The vendor-specific functions might have changed features.
  734. * Now we do "generic changes."
  735. */
  736. /* Filter out anything that depends on CPUID levels we don't have */
  737. filter_cpuid_features(c, true);
  738. /* If the model name is still unset, do table lookup. */
  739. if (!c->x86_model_id[0]) {
  740. const char *p;
  741. p = table_lookup_model(c);
  742. if (p)
  743. strcpy(c->x86_model_id, p);
  744. else
  745. /* Last resort... */
  746. sprintf(c->x86_model_id, "%02x/%02x",
  747. c->x86, c->x86_model);
  748. }
  749. #ifdef CONFIG_X86_64
  750. detect_ht(c);
  751. #endif
  752. init_hypervisor(c);
  753. x86_init_rdrand(c);
  754. /*
  755. * Clear/Set all flags overriden by options, need do it
  756. * before following smp all cpus cap AND.
  757. */
  758. for (i = 0; i < NCAPINTS; i++) {
  759. c->x86_capability[i] &= ~cpu_caps_cleared[i];
  760. c->x86_capability[i] |= cpu_caps_set[i];
  761. }
  762. /*
  763. * On SMP, boot_cpu_data holds the common feature set between
  764. * all CPUs; so make sure that we indicate which features are
  765. * common between the CPUs. The first time this routine gets
  766. * executed, c == &boot_cpu_data.
  767. */
  768. if (c != &boot_cpu_data) {
  769. /* AND the already accumulated flags with these */
  770. for (i = 0; i < NCAPINTS; i++)
  771. boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
  772. /* OR, i.e. replicate the bug flags */
  773. for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
  774. c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
  775. }
  776. /* Init Machine Check Exception if available. */
  777. mcheck_cpu_init(c);
  778. select_idle_routine(c);
  779. #ifdef CONFIG_NUMA
  780. numa_add_cpu(smp_processor_id());
  781. #endif
  782. }
  783. #ifdef CONFIG_X86_64
  784. static void vgetcpu_set_mode(void)
  785. {
  786. if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
  787. vgetcpu_mode = VGETCPU_RDTSCP;
  788. else
  789. vgetcpu_mode = VGETCPU_LSL;
  790. }
  791. #endif
  792. void __init identify_boot_cpu(void)
  793. {
  794. identify_cpu(&boot_cpu_data);
  795. init_amd_e400_c1e_mask();
  796. #ifdef CONFIG_X86_32
  797. sysenter_setup();
  798. enable_sep_cpu();
  799. #else
  800. vgetcpu_set_mode();
  801. #endif
  802. cpu_detect_tlb(&boot_cpu_data);
  803. }
  804. void identify_secondary_cpu(struct cpuinfo_x86 *c)
  805. {
  806. BUG_ON(c == &boot_cpu_data);
  807. identify_cpu(c);
  808. #ifdef CONFIG_X86_32
  809. enable_sep_cpu();
  810. #endif
  811. mtrr_ap_init();
  812. }
  813. struct msr_range {
  814. unsigned min;
  815. unsigned max;
  816. };
  817. static const struct msr_range msr_range_array[] = {
  818. { 0x00000000, 0x00000418},
  819. { 0xc0000000, 0xc000040b},
  820. { 0xc0010000, 0xc0010142},
  821. { 0xc0011000, 0xc001103b},
  822. };
  823. static void __print_cpu_msr(void)
  824. {
  825. unsigned index_min, index_max;
  826. unsigned index;
  827. u64 val;
  828. int i;
  829. for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
  830. index_min = msr_range_array[i].min;
  831. index_max = msr_range_array[i].max;
  832. for (index = index_min; index < index_max; index++) {
  833. if (rdmsrl_safe(index, &val))
  834. continue;
  835. printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
  836. }
  837. }
  838. }
  839. static int show_msr;
  840. static __init int setup_show_msr(char *arg)
  841. {
  842. int num;
  843. get_option(&arg, &num);
  844. if (num > 0)
  845. show_msr = num;
  846. return 1;
  847. }
  848. __setup("show_msr=", setup_show_msr);
  849. static __init int setup_noclflush(char *arg)
  850. {
  851. setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
  852. return 1;
  853. }
  854. __setup("noclflush", setup_noclflush);
  855. void print_cpu_info(struct cpuinfo_x86 *c)
  856. {
  857. const char *vendor = NULL;
  858. if (c->x86_vendor < X86_VENDOR_NUM) {
  859. vendor = this_cpu->c_vendor;
  860. } else {
  861. if (c->cpuid_level >= 0)
  862. vendor = c->x86_vendor_id;
  863. }
  864. if (vendor && !strstr(c->x86_model_id, vendor))
  865. printk(KERN_CONT "%s ", vendor);
  866. if (c->x86_model_id[0])
  867. printk(KERN_CONT "%s", strim(c->x86_model_id));
  868. else
  869. printk(KERN_CONT "%d86", c->x86);
  870. printk(KERN_CONT " (fam: %02x, model: %02x", c->x86, c->x86_model);
  871. if (c->x86_mask || c->cpuid_level >= 0)
  872. printk(KERN_CONT ", stepping: %02x)\n", c->x86_mask);
  873. else
  874. printk(KERN_CONT ")\n");
  875. print_cpu_msr(c);
  876. }
  877. void print_cpu_msr(struct cpuinfo_x86 *c)
  878. {
  879. if (c->cpu_index < show_msr)
  880. __print_cpu_msr();
  881. }
  882. static __init int setup_disablecpuid(char *arg)
  883. {
  884. int bit;
  885. if (get_option(&arg, &bit) && bit < NCAPINTS*32)
  886. setup_clear_cpu_cap(bit);
  887. else
  888. return 0;
  889. return 1;
  890. }
  891. __setup("clearcpuid=", setup_disablecpuid);
  892. #ifdef CONFIG_X86_64
  893. struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
  894. struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
  895. (unsigned long) debug_idt_table };
  896. DEFINE_PER_CPU_FIRST(union irq_stack_union,
  897. irq_stack_union) __aligned(PAGE_SIZE);
  898. /*
  899. * The following four percpu variables are hot. Align current_task to
  900. * cacheline size such that all four fall in the same cacheline.
  901. */
  902. DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
  903. &init_task;
  904. EXPORT_PER_CPU_SYMBOL(current_task);
  905. DEFINE_PER_CPU(unsigned long, kernel_stack) =
  906. (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
  907. EXPORT_PER_CPU_SYMBOL(kernel_stack);
  908. DEFINE_PER_CPU(char *, irq_stack_ptr) =
  909. init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
  910. DEFINE_PER_CPU(unsigned int, irq_count) = -1;
  911. DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
  912. /*
  913. * Special IST stacks which the CPU switches to when it calls
  914. * an IST-marked descriptor entry. Up to 7 stacks (hardware
  915. * limit), all of them are 4K, except the debug stack which
  916. * is 8K.
  917. */
  918. static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
  919. [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ,
  920. [DEBUG_STACK - 1] = DEBUG_STKSZ
  921. };
  922. static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
  923. [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
  924. /* May not be marked __init: used by software suspend */
  925. void syscall_init(void)
  926. {
  927. /*
  928. * LSTAR and STAR live in a bit strange symbiosis.
  929. * They both write to the same internal register. STAR allows to
  930. * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
  931. */
  932. wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 | ((u64)__KERNEL_CS)<<32);
  933. wrmsrl(MSR_LSTAR, system_call);
  934. wrmsrl(MSR_CSTAR, ignore_sysret);
  935. #ifdef CONFIG_IA32_EMULATION
  936. syscall32_cpu_init();
  937. #endif
  938. /* Flags to clear on syscall */
  939. wrmsrl(MSR_SYSCALL_MASK,
  940. X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
  941. X86_EFLAGS_IOPL|X86_EFLAGS_AC);
  942. }
  943. /*
  944. * Copies of the original ist values from the tss are only accessed during
  945. * debugging, no special alignment required.
  946. */
  947. DEFINE_PER_CPU(struct orig_ist, orig_ist);
  948. static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
  949. DEFINE_PER_CPU(int, debug_stack_usage);
  950. int is_debug_stack(unsigned long addr)
  951. {
  952. return __get_cpu_var(debug_stack_usage) ||
  953. (addr <= __get_cpu_var(debug_stack_addr) &&
  954. addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
  955. }
  956. DEFINE_PER_CPU(u32, debug_idt_ctr);
  957. void debug_stack_set_zero(void)
  958. {
  959. this_cpu_inc(debug_idt_ctr);
  960. load_current_idt();
  961. }
  962. void debug_stack_reset(void)
  963. {
  964. if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
  965. return;
  966. if (this_cpu_dec_return(debug_idt_ctr) == 0)
  967. load_current_idt();
  968. }
  969. #else /* CONFIG_X86_64 */
  970. DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
  971. EXPORT_PER_CPU_SYMBOL(current_task);
  972. DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
  973. #ifdef CONFIG_CC_STACKPROTECTOR
  974. DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
  975. #endif
  976. #endif /* CONFIG_X86_64 */
  977. /*
  978. * Clear all 6 debug registers:
  979. */
  980. static void clear_all_debug_regs(void)
  981. {
  982. int i;
  983. for (i = 0; i < 8; i++) {
  984. /* Ignore db4, db5 */
  985. if ((i == 4) || (i == 5))
  986. continue;
  987. set_debugreg(0, i);
  988. }
  989. }
  990. #ifdef CONFIG_KGDB
  991. /*
  992. * Restore debug regs if using kgdbwait and you have a kernel debugger
  993. * connection established.
  994. */
  995. static void dbg_restore_debug_regs(void)
  996. {
  997. if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
  998. arch_kgdb_ops.correct_hw_break();
  999. }
  1000. #else /* ! CONFIG_KGDB */
  1001. #define dbg_restore_debug_regs()
  1002. #endif /* ! CONFIG_KGDB */
  1003. /*
  1004. * cpu_init() initializes state that is per-CPU. Some data is already
  1005. * initialized (naturally) in the bootstrap process, such as the GDT
  1006. * and IDT. We reload them nevertheless, this function acts as a
  1007. * 'CPU state barrier', nothing should get across.
  1008. * A lot of state is already set up in PDA init for 64 bit
  1009. */
  1010. #ifdef CONFIG_X86_64
  1011. void cpu_init(void)
  1012. {
  1013. struct orig_ist *oist;
  1014. struct task_struct *me;
  1015. struct tss_struct *t;
  1016. unsigned long v;
  1017. int cpu;
  1018. int i;
  1019. /*
  1020. * Load microcode on this cpu if a valid microcode is available.
  1021. * This is early microcode loading procedure.
  1022. */
  1023. load_ucode_ap();
  1024. cpu = stack_smp_processor_id();
  1025. t = &per_cpu(init_tss, cpu);
  1026. oist = &per_cpu(orig_ist, cpu);
  1027. #ifdef CONFIG_NUMA
  1028. if (this_cpu_read(numa_node) == 0 &&
  1029. early_cpu_to_node(cpu) != NUMA_NO_NODE)
  1030. set_numa_node(early_cpu_to_node(cpu));
  1031. #endif
  1032. me = current;
  1033. if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask))
  1034. panic("CPU#%d already initialized!\n", cpu);
  1035. pr_debug("Initializing CPU#%d\n", cpu);
  1036. clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
  1037. /*
  1038. * Initialize the per-CPU GDT with the boot GDT,
  1039. * and set up the GDT descriptor:
  1040. */
  1041. switch_to_new_gdt(cpu);
  1042. loadsegment(fs, 0);
  1043. load_current_idt();
  1044. memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
  1045. syscall_init();
  1046. wrmsrl(MSR_FS_BASE, 0);
  1047. wrmsrl(MSR_KERNEL_GS_BASE, 0);
  1048. barrier();
  1049. x86_configure_nx();
  1050. enable_x2apic();
  1051. /*
  1052. * set up and load the per-CPU TSS
  1053. */
  1054. if (!oist->ist[0]) {
  1055. char *estacks = per_cpu(exception_stacks, cpu);
  1056. for (v = 0; v < N_EXCEPTION_STACKS; v++) {
  1057. estacks += exception_stack_sizes[v];
  1058. oist->ist[v] = t->x86_tss.ist[v] =
  1059. (unsigned long)estacks;
  1060. if (v == DEBUG_STACK-1)
  1061. per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
  1062. }
  1063. }
  1064. t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
  1065. /*
  1066. * <= is required because the CPU will access up to
  1067. * 8 bits beyond the end of the IO permission bitmap.
  1068. */
  1069. for (i = 0; i <= IO_BITMAP_LONGS; i++)
  1070. t->io_bitmap[i] = ~0UL;
  1071. atomic_inc(&init_mm.mm_count);
  1072. me->active_mm = &init_mm;
  1073. BUG_ON(me->mm);
  1074. enter_lazy_tlb(&init_mm, me);
  1075. load_sp0(t, &current->thread);
  1076. set_tss_desc(cpu, t);
  1077. load_TR_desc();
  1078. load_LDT(&init_mm.context);
  1079. clear_all_debug_regs();
  1080. dbg_restore_debug_regs();
  1081. fpu_init();
  1082. if (is_uv_system())
  1083. uv_cpu_init();
  1084. }
  1085. #else
  1086. void cpu_init(void)
  1087. {
  1088. int cpu = smp_processor_id();
  1089. struct task_struct *curr = current;
  1090. struct tss_struct *t = &per_cpu(init_tss, cpu);
  1091. struct thread_struct *thread = &curr->thread;
  1092. show_ucode_info_early();
  1093. if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
  1094. printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
  1095. for (;;)
  1096. local_irq_enable();
  1097. }
  1098. printk(KERN_INFO "Initializing CPU#%d\n", cpu);
  1099. if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
  1100. clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
  1101. load_current_idt();
  1102. switch_to_new_gdt(cpu);
  1103. /*
  1104. * Set up and load the per-CPU TSS and LDT
  1105. */
  1106. atomic_inc(&init_mm.mm_count);
  1107. curr->active_mm = &init_mm;
  1108. BUG_ON(curr->mm);
  1109. enter_lazy_tlb(&init_mm, curr);
  1110. load_sp0(t, thread);
  1111. set_tss_desc(cpu, t);
  1112. load_TR_desc();
  1113. load_LDT(&init_mm.context);
  1114. t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
  1115. #ifdef CONFIG_DOUBLEFAULT
  1116. /* Set up doublefault TSS pointer in the GDT */
  1117. __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
  1118. #endif
  1119. clear_all_debug_regs();
  1120. dbg_restore_debug_regs();
  1121. fpu_init();
  1122. }
  1123. #endif
  1124. #ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
  1125. void warn_pre_alternatives(void)
  1126. {
  1127. WARN(1, "You're using static_cpu_has before alternatives have run!\n");
  1128. }
  1129. EXPORT_SYMBOL_GPL(warn_pre_alternatives);
  1130. #endif
  1131. inline bool __static_cpu_has_safe(u16 bit)
  1132. {
  1133. return boot_cpu_has(bit);
  1134. }
  1135. EXPORT_SYMBOL_GPL(__static_cpu_has_safe);