generic.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. /*
  2. * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
  3. * because MTRRs can span upto 40 bits (36bits on most modern x86)
  4. */
  5. #define DEBUG
  6. #include <linux/module.h>
  7. #include <linux/init.h>
  8. #include <linux/io.h>
  9. #include <linux/mm.h>
  10. #include <asm/processor-flags.h>
  11. #include <asm/cpufeature.h>
  12. #include <asm/tlbflush.h>
  13. #include <asm/system.h>
  14. #include <asm/mtrr.h>
  15. #include <asm/msr.h>
  16. #include <asm/pat.h>
  17. #include "mtrr.h"
  18. struct fixed_range_block {
  19. int base_msr; /* start address of an MTRR block */
  20. int ranges; /* number of MTRRs in this block */
  21. };
  22. static struct fixed_range_block fixed_range_blocks[] = {
  23. { MSR_MTRRfix64K_00000, 1 }, /* one 64k MTRR */
  24. { MSR_MTRRfix16K_80000, 2 }, /* two 16k MTRRs */
  25. { MSR_MTRRfix4K_C0000, 8 }, /* eight 4k MTRRs */
  26. {}
  27. };
  28. static unsigned long smp_changes_mask;
  29. static int mtrr_state_set;
  30. u64 mtrr_tom2;
  31. struct mtrr_state_type mtrr_state;
  32. EXPORT_SYMBOL_GPL(mtrr_state);
  33. /*
  34. * BIOS is expected to clear MtrrFixDramModEn bit, see for example
  35. * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
  36. * Opteron Processors" (26094 Rev. 3.30 February 2006), section
  37. * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
  38. * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
  39. * 0 for operation."
  40. */
  41. static inline void k8_check_syscfg_dram_mod_en(void)
  42. {
  43. u32 lo, hi;
  44. if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
  45. (boot_cpu_data.x86 >= 0x0f)))
  46. return;
  47. rdmsr(MSR_K8_SYSCFG, lo, hi);
  48. if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
  49. printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
  50. " not cleared by BIOS, clearing this bit\n",
  51. smp_processor_id());
  52. lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
  53. mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
  54. }
  55. }
  56. /*
  57. * Returns the effective MTRR type for the region
  58. * Error returns:
  59. * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
  60. * - 0xFF - when MTRR is not enabled
  61. */
  62. u8 mtrr_type_lookup(u64 start, u64 end)
  63. {
  64. int i;
  65. u64 base, mask;
  66. u8 prev_match, curr_match;
  67. if (!mtrr_state_set)
  68. return 0xFF;
  69. if (!mtrr_state.enabled)
  70. return 0xFF;
  71. /* Make end inclusive end, instead of exclusive */
  72. end--;
  73. /* Look in fixed ranges. Just return the type as per start */
  74. if (mtrr_state.have_fixed && (start < 0x100000)) {
  75. int idx;
  76. if (start < 0x80000) {
  77. idx = 0;
  78. idx += (start >> 16);
  79. return mtrr_state.fixed_ranges[idx];
  80. } else if (start < 0xC0000) {
  81. idx = 1 * 8;
  82. idx += ((start - 0x80000) >> 14);
  83. return mtrr_state.fixed_ranges[idx];
  84. } else if (start < 0x1000000) {
  85. idx = 3 * 8;
  86. idx += ((start - 0xC0000) >> 12);
  87. return mtrr_state.fixed_ranges[idx];
  88. }
  89. }
  90. /*
  91. * Look in variable ranges
  92. * Look of multiple ranges matching this address and pick type
  93. * as per MTRR precedence
  94. */
  95. if (!(mtrr_state.enabled & 2))
  96. return mtrr_state.def_type;
  97. prev_match = 0xFF;
  98. for (i = 0; i < num_var_ranges; ++i) {
  99. unsigned short start_state, end_state;
  100. if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
  101. continue;
  102. base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
  103. (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
  104. mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
  105. (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);
  106. start_state = ((start & mask) == (base & mask));
  107. end_state = ((end & mask) == (base & mask));
  108. if (start_state != end_state)
  109. return 0xFE;
  110. if ((start & mask) != (base & mask))
  111. continue;
  112. curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
  113. if (prev_match == 0xFF) {
  114. prev_match = curr_match;
  115. continue;
  116. }
  117. if (prev_match == MTRR_TYPE_UNCACHABLE ||
  118. curr_match == MTRR_TYPE_UNCACHABLE) {
  119. return MTRR_TYPE_UNCACHABLE;
  120. }
  121. if ((prev_match == MTRR_TYPE_WRBACK &&
  122. curr_match == MTRR_TYPE_WRTHROUGH) ||
  123. (prev_match == MTRR_TYPE_WRTHROUGH &&
  124. curr_match == MTRR_TYPE_WRBACK)) {
  125. prev_match = MTRR_TYPE_WRTHROUGH;
  126. curr_match = MTRR_TYPE_WRTHROUGH;
  127. }
  128. if (prev_match != curr_match)
  129. return MTRR_TYPE_UNCACHABLE;
  130. }
  131. if (mtrr_tom2) {
  132. if (start >= (1ULL<<32) && (end < mtrr_tom2))
  133. return MTRR_TYPE_WRBACK;
  134. }
  135. if (prev_match != 0xFF)
  136. return prev_match;
  137. return mtrr_state.def_type;
  138. }
  139. /* Get the MSR pair relating to a var range */
  140. static void
  141. get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
  142. {
  143. rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
  144. rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
  145. }
  146. /* Fill the MSR pair relating to a var range */
  147. void fill_mtrr_var_range(unsigned int index,
  148. u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
  149. {
  150. struct mtrr_var_range *vr;
  151. vr = mtrr_state.var_ranges;
  152. vr[index].base_lo = base_lo;
  153. vr[index].base_hi = base_hi;
  154. vr[index].mask_lo = mask_lo;
  155. vr[index].mask_hi = mask_hi;
  156. }
  157. static void get_fixed_ranges(mtrr_type *frs)
  158. {
  159. unsigned int *p = (unsigned int *)frs;
  160. int i;
  161. k8_check_syscfg_dram_mod_en();
  162. rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
  163. for (i = 0; i < 2; i++)
  164. rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
  165. for (i = 0; i < 8; i++)
  166. rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
  167. }
  168. void mtrr_save_fixed_ranges(void *info)
  169. {
  170. if (cpu_has_mtrr)
  171. get_fixed_ranges(mtrr_state.fixed_ranges);
  172. }
  173. static unsigned __initdata last_fixed_start;
  174. static unsigned __initdata last_fixed_end;
  175. static mtrr_type __initdata last_fixed_type;
  176. static void __init print_fixed_last(void)
  177. {
  178. if (!last_fixed_end)
  179. return;
  180. pr_debug(" %05X-%05X %s\n", last_fixed_start,
  181. last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
  182. last_fixed_end = 0;
  183. }
  184. static void __init update_fixed_last(unsigned base, unsigned end,
  185. mtrr_type type)
  186. {
  187. last_fixed_start = base;
  188. last_fixed_end = end;
  189. last_fixed_type = type;
  190. }
  191. static void __init
  192. print_fixed(unsigned base, unsigned step, const mtrr_type *types)
  193. {
  194. unsigned i;
  195. for (i = 0; i < 8; ++i, ++types, base += step) {
  196. if (last_fixed_end == 0) {
  197. update_fixed_last(base, base + step, *types);
  198. continue;
  199. }
  200. if (last_fixed_end == base && last_fixed_type == *types) {
  201. last_fixed_end = base + step;
  202. continue;
  203. }
  204. /* new segments: gap or different type */
  205. print_fixed_last();
  206. update_fixed_last(base, base + step, *types);
  207. }
  208. }
  209. static void prepare_set(void);
  210. static void post_set(void);
  211. static void __init print_mtrr_state(void)
  212. {
  213. unsigned int i;
  214. int high_width;
  215. pr_debug("MTRR default type: %s\n",
  216. mtrr_attrib_to_str(mtrr_state.def_type));
  217. if (mtrr_state.have_fixed) {
  218. pr_debug("MTRR fixed ranges %sabled:\n",
  219. mtrr_state.enabled & 1 ? "en" : "dis");
  220. print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
  221. for (i = 0; i < 2; ++i)
  222. print_fixed(0x80000 + i * 0x20000, 0x04000,
  223. mtrr_state.fixed_ranges + (i + 1) * 8);
  224. for (i = 0; i < 8; ++i)
  225. print_fixed(0xC0000 + i * 0x08000, 0x01000,
  226. mtrr_state.fixed_ranges + (i + 3) * 8);
  227. /* tail */
  228. print_fixed_last();
  229. }
  230. pr_debug("MTRR variable ranges %sabled:\n",
  231. mtrr_state.enabled & 2 ? "en" : "dis");
  232. if (size_or_mask & 0xffffffffUL)
  233. high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
  234. else
  235. high_width = ffs(size_or_mask>>32) + 32 - 1;
  236. high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
  237. for (i = 0; i < num_var_ranges; ++i) {
  238. if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
  239. pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n",
  240. i,
  241. high_width,
  242. mtrr_state.var_ranges[i].base_hi,
  243. mtrr_state.var_ranges[i].base_lo >> 12,
  244. high_width,
  245. mtrr_state.var_ranges[i].mask_hi,
  246. mtrr_state.var_ranges[i].mask_lo >> 12,
  247. mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
  248. else
  249. pr_debug(" %u disabled\n", i);
  250. }
  251. if (mtrr_tom2)
  252. pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
  253. }
  254. /* Grab all of the MTRR state for this CPU into *state */
  255. void __init get_mtrr_state(void)
  256. {
  257. struct mtrr_var_range *vrs;
  258. unsigned long flags;
  259. unsigned lo, dummy;
  260. unsigned int i;
  261. vrs = mtrr_state.var_ranges;
  262. rdmsr(MSR_MTRRcap, lo, dummy);
  263. mtrr_state.have_fixed = (lo >> 8) & 1;
  264. for (i = 0; i < num_var_ranges; i++)
  265. get_mtrr_var_range(i, &vrs[i]);
  266. if (mtrr_state.have_fixed)
  267. get_fixed_ranges(mtrr_state.fixed_ranges);
  268. rdmsr(MSR_MTRRdefType, lo, dummy);
  269. mtrr_state.def_type = (lo & 0xff);
  270. mtrr_state.enabled = (lo & 0xc00) >> 10;
  271. if (amd_special_default_mtrr()) {
  272. unsigned low, high;
  273. /* TOP_MEM2 */
  274. rdmsr(MSR_K8_TOP_MEM2, low, high);
  275. mtrr_tom2 = high;
  276. mtrr_tom2 <<= 32;
  277. mtrr_tom2 |= low;
  278. mtrr_tom2 &= 0xffffff800000ULL;
  279. }
  280. print_mtrr_state();
  281. mtrr_state_set = 1;
  282. /* PAT setup for BP. We need to go through sync steps here */
  283. local_irq_save(flags);
  284. prepare_set();
  285. pat_init();
  286. post_set();
  287. local_irq_restore(flags);
  288. }
  289. /* Some BIOS's are messed up and don't set all MTRRs the same! */
  290. void __init mtrr_state_warn(void)
  291. {
  292. unsigned long mask = smp_changes_mask;
  293. if (!mask)
  294. return;
  295. if (mask & MTRR_CHANGE_MASK_FIXED)
  296. pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
  297. if (mask & MTRR_CHANGE_MASK_VARIABLE)
  298. pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
  299. if (mask & MTRR_CHANGE_MASK_DEFTYPE)
  300. pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");
  301. printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
  302. printk(KERN_INFO "mtrr: corrected configuration.\n");
  303. }
  304. /*
  305. * Doesn't attempt to pass an error out to MTRR users
  306. * because it's quite complicated in some cases and probably not
  307. * worth it because the best error handling is to ignore it.
  308. */
  309. void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
  310. {
  311. if (wrmsr_safe(msr, a, b) < 0) {
  312. printk(KERN_ERR
  313. "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
  314. smp_processor_id(), msr, a, b);
  315. }
  316. }
  317. /**
  318. * set_fixed_range - checks & updates a fixed-range MTRR if it
  319. * differs from the value it should have
  320. * @msr: MSR address of the MTTR which should be checked and updated
  321. * @changed: pointer which indicates whether the MTRR needed to be changed
  322. * @msrwords: pointer to the MSR values which the MSR should have
  323. */
  324. static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
  325. {
  326. unsigned lo, hi;
  327. rdmsr(msr, lo, hi);
  328. if (lo != msrwords[0] || hi != msrwords[1]) {
  329. mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
  330. *changed = true;
  331. }
  332. }
  333. /**
  334. * generic_get_free_region - Get a free MTRR.
  335. * @base: The starting (base) address of the region.
  336. * @size: The size (in bytes) of the region.
  337. * @replace_reg: mtrr index to be replaced; set to invalid value if none.
  338. *
  339. * Returns: The index of the region on success, else negative on error.
  340. */
  341. int
  342. generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
  343. {
  344. unsigned long lbase, lsize;
  345. mtrr_type ltype;
  346. int i, max;
  347. max = num_var_ranges;
  348. if (replace_reg >= 0 && replace_reg < max)
  349. return replace_reg;
  350. for (i = 0; i < max; ++i) {
  351. mtrr_if->get(i, &lbase, &lsize, &ltype);
  352. if (lsize == 0)
  353. return i;
  354. }
  355. return -ENOSPC;
  356. }
  357. static void generic_get_mtrr(unsigned int reg, unsigned long *base,
  358. unsigned long *size, mtrr_type *type)
  359. {
  360. unsigned int mask_lo, mask_hi, base_lo, base_hi;
  361. unsigned int tmp, hi;
  362. int cpu;
  363. /*
  364. * get_mtrr doesn't need to update mtrr_state, also it could be called
  365. * from any cpu, so try to print it out directly.
  366. */
  367. cpu = get_cpu();
  368. rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
  369. if ((mask_lo & 0x800) == 0) {
  370. /* Invalid (i.e. free) range */
  371. *base = 0;
  372. *size = 0;
  373. *type = 0;
  374. goto out_put_cpu;
  375. }
  376. rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
  377. /* Work out the shifted address mask: */
  378. tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
  379. mask_lo = size_or_mask | tmp;
  380. /* Expand tmp with high bits to all 1s: */
  381. hi = fls(tmp);
  382. if (hi > 0) {
  383. tmp |= ~((1<<(hi - 1)) - 1);
  384. if (tmp != mask_lo) {
  385. printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
  386. mask_lo = tmp;
  387. }
  388. }
  389. /*
  390. * This works correctly if size is a power of two, i.e. a
  391. * contiguous range:
  392. */
  393. *size = -mask_lo;
  394. *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
  395. *type = base_lo & 0xff;
  396. out_put_cpu:
  397. put_cpu();
  398. }
  399. /**
  400. * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
  401. * differ from the saved set
  402. * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
  403. */
  404. static int set_fixed_ranges(mtrr_type *frs)
  405. {
  406. unsigned long long *saved = (unsigned long long *)frs;
  407. bool changed = false;
  408. int block = -1, range;
  409. k8_check_syscfg_dram_mod_en();
  410. while (fixed_range_blocks[++block].ranges) {
  411. for (range = 0; range < fixed_range_blocks[block].ranges; range++)
  412. set_fixed_range(fixed_range_blocks[block].base_msr + range,
  413. &changed, (unsigned int *)saved++);
  414. }
  415. return changed;
  416. }
  417. /*
  418. * Set the MSR pair relating to a var range.
  419. * Returns true if changes are made.
  420. */
  421. static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
  422. {
  423. unsigned int lo, hi;
  424. bool changed = false;
  425. rdmsr(MTRRphysBase_MSR(index), lo, hi);
  426. if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
  427. || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
  428. (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
  429. mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
  430. changed = true;
  431. }
  432. rdmsr(MTRRphysMask_MSR(index), lo, hi);
  433. if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
  434. || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
  435. (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
  436. mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
  437. changed = true;
  438. }
  439. return changed;
  440. }
  441. static u32 deftype_lo, deftype_hi;
  442. /**
  443. * set_mtrr_state - Set the MTRR state for this CPU.
  444. *
  445. * NOTE: The CPU must already be in a safe state for MTRR changes.
  446. * RETURNS: 0 if no changes made, else a mask indicating what was changed.
  447. */
  448. static unsigned long set_mtrr_state(void)
  449. {
  450. unsigned long change_mask = 0;
  451. unsigned int i;
  452. for (i = 0; i < num_var_ranges; i++) {
  453. if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
  454. change_mask |= MTRR_CHANGE_MASK_VARIABLE;
  455. }
  456. if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
  457. change_mask |= MTRR_CHANGE_MASK_FIXED;
  458. /*
  459. * Set_mtrr_restore restores the old value of MTRRdefType,
  460. * so to set it we fiddle with the saved value:
  461. */
  462. if ((deftype_lo & 0xff) != mtrr_state.def_type
  463. || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
  464. deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
  465. (mtrr_state.enabled << 10);
  466. change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
  467. }
  468. return change_mask;
  469. }
  470. static unsigned long cr4;
  471. static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
  472. /*
  473. * Since we are disabling the cache don't allow any interrupts,
  474. * they would run extremely slow and would only increase the pain.
  475. *
  476. * The caller must ensure that local interrupts are disabled and
  477. * are reenabled after post_set() has been called.
  478. */
  479. static void prepare_set(void) __acquires(set_atomicity_lock)
  480. {
  481. unsigned long cr0;
  482. /*
  483. * Note that this is not ideal
  484. * since the cache is only flushed/disabled for this CPU while the
  485. * MTRRs are changed, but changing this requires more invasive
  486. * changes to the way the kernel boots
  487. */
  488. raw_spin_lock(&set_atomicity_lock);
  489. /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
  490. cr0 = read_cr0() | X86_CR0_CD;
  491. write_cr0(cr0);
  492. wbinvd();
  493. /* Save value of CR4 and clear Page Global Enable (bit 7) */
  494. if (cpu_has_pge) {
  495. cr4 = read_cr4();
  496. write_cr4(cr4 & ~X86_CR4_PGE);
  497. }
  498. /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
  499. __flush_tlb();
  500. /* Save MTRR state */
  501. rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
  502. /* Disable MTRRs, and set the default type to uncached */
  503. mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
  504. }
  505. static void post_set(void) __releases(set_atomicity_lock)
  506. {
  507. /* Flush TLBs (no need to flush caches - they are disabled) */
  508. __flush_tlb();
  509. /* Intel (P6) standard MTRRs */
  510. mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
  511. /* Enable caches */
  512. write_cr0(read_cr0() & 0xbfffffff);
  513. /* Restore value of CR4 */
  514. if (cpu_has_pge)
  515. write_cr4(cr4);
  516. raw_spin_unlock(&set_atomicity_lock);
  517. }
  518. static void generic_set_all(void)
  519. {
  520. unsigned long mask, count;
  521. unsigned long flags;
  522. local_irq_save(flags);
  523. prepare_set();
  524. /* Actually set the state */
  525. mask = set_mtrr_state();
  526. /* also set PAT */
  527. pat_init();
  528. post_set();
  529. local_irq_restore(flags);
  530. /* Use the atomic bitops to update the global mask */
  531. for (count = 0; count < sizeof mask * 8; ++count) {
  532. if (mask & 0x01)
  533. set_bit(count, &smp_changes_mask);
  534. mask >>= 1;
  535. }
  536. }
  537. /**
  538. * generic_set_mtrr - set variable MTRR register on the local CPU.
  539. *
  540. * @reg: The register to set.
  541. * @base: The base address of the region.
  542. * @size: The size of the region. If this is 0 the region is disabled.
  543. * @type: The type of the region.
  544. *
  545. * Returns nothing.
  546. */
  547. static void generic_set_mtrr(unsigned int reg, unsigned long base,
  548. unsigned long size, mtrr_type type)
  549. {
  550. unsigned long flags;
  551. struct mtrr_var_range *vr;
  552. vr = &mtrr_state.var_ranges[reg];
  553. local_irq_save(flags);
  554. prepare_set();
  555. if (size == 0) {
  556. /*
  557. * The invalid bit is kept in the mask, so we simply
  558. * clear the relevant mask register to disable a range.
  559. */
  560. mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
  561. memset(vr, 0, sizeof(struct mtrr_var_range));
  562. } else {
  563. vr->base_lo = base << PAGE_SHIFT | type;
  564. vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
  565. vr->mask_lo = -size << PAGE_SHIFT | 0x800;
  566. vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
  567. mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
  568. mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
  569. }
  570. post_set();
  571. local_irq_restore(flags);
  572. }
  573. int generic_validate_add_page(unsigned long base, unsigned long size,
  574. unsigned int type)
  575. {
  576. unsigned long lbase, last;
  577. /*
  578. * For Intel PPro stepping <= 7
  579. * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
  580. */
  581. if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
  582. boot_cpu_data.x86_model == 1 &&
  583. boot_cpu_data.x86_mask <= 7) {
  584. if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
  585. pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
  586. return -EINVAL;
  587. }
  588. if (!(base + size < 0x70000 || base > 0x7003F) &&
  589. (type == MTRR_TYPE_WRCOMB
  590. || type == MTRR_TYPE_WRBACK)) {
  591. pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
  592. return -EINVAL;
  593. }
  594. }
  595. /*
  596. * Check upper bits of base and last are equal and lower bits are 0
  597. * for base and 1 for last
  598. */
  599. last = base + size - 1;
  600. for (lbase = base; !(lbase & 1) && (last & 1);
  601. lbase = lbase >> 1, last = last >> 1)
  602. ;
  603. if (lbase != last) {
  604. pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
  605. return -EINVAL;
  606. }
  607. return 0;
  608. }
  609. static int generic_have_wrcomb(void)
  610. {
  611. unsigned long config, dummy;
  612. rdmsr(MSR_MTRRcap, config, dummy);
  613. return config & (1 << 10);
  614. }
  615. int positive_have_wrcomb(void)
  616. {
  617. return 1;
  618. }
  619. /*
  620. * Generic structure...
  621. */
  622. const struct mtrr_ops generic_mtrr_ops = {
  623. .use_intel_if = 1,
  624. .set_all = generic_set_all,
  625. .get = generic_get_mtrr,
  626. .get_free_region = generic_get_free_region,
  627. .set = generic_set_mtrr,
  628. .validate_add_page = generic_validate_add_page,
  629. .have_wrcomb = generic_have_wrcomb,
  630. };