main.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807
  1. /* Generic MTRR (Memory Type Range Register) driver.
  2. Copyright (C) 1997-2000 Richard Gooch
  3. Copyright (c) 2002 Patrick Mochel
  4. This library is free software; you can redistribute it and/or
  5. modify it under the terms of the GNU Library General Public
  6. License as published by the Free Software Foundation; either
  7. version 2 of the License, or (at your option) any later version.
  8. This library is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. Library General Public License for more details.
  12. You should have received a copy of the GNU Library General Public
  13. License along with this library; if not, write to the Free
  14. Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  15. Richard Gooch may be reached by email at rgooch@atnf.csiro.au
  16. The postal address is:
  17. Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
  18. Source: "Pentium Pro Family Developer's Manual, Volume 3:
  19. Operating System Writer's Guide" (Intel document number 242692),
  20. section 11.11.7
  21. This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
  22. on 6-7 March 2002.
  23. Source: Intel Architecture Software Developers Manual, Volume 3:
  24. System Programming Guide; Section 9.11. (1997 edition - PPro).
  25. */
  26. #define DEBUG
  27. #include <linux/types.h> /* FIXME: kvm_para.h needs this */
  28. #include <linux/kvm_para.h>
  29. #include <linux/uaccess.h>
  30. #include <linux/module.h>
  31. #include <linux/mutex.h>
  32. #include <linux/init.h>
  33. #include <linux/sort.h>
  34. #include <linux/cpu.h>
  35. #include <linux/pci.h>
  36. #include <linux/smp.h>
  37. #include <asm/processor.h>
  38. #include <asm/e820.h>
  39. #include <asm/mtrr.h>
  40. #include <asm/msr.h>
  41. #include "mtrr.h"
  42. u32 num_var_ranges;
  43. unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
  44. static DEFINE_MUTEX(mtrr_mutex);
  45. u64 size_or_mask, size_and_mask;
  46. static bool mtrr_aps_delayed_init;
  47. static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
  48. struct mtrr_ops *mtrr_if;
  49. static void set_mtrr(unsigned int reg, unsigned long base,
  50. unsigned long size, mtrr_type type);
  51. void set_mtrr_ops(struct mtrr_ops *ops)
  52. {
  53. if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
  54. mtrr_ops[ops->vendor] = ops;
  55. }
  56. /* Returns non-zero if we have the write-combining memory type */
  57. static int have_wrcomb(void)
  58. {
  59. struct pci_dev *dev;
  60. u8 rev;
  61. dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
  62. if (dev != NULL) {
  63. /*
  64. * ServerWorks LE chipsets < rev 6 have problems with
  65. * write-combining. Don't allow it and leave room for other
  66. * chipsets to be tagged
  67. */
  68. if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
  69. dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) {
  70. pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev);
  71. if (rev <= 5) {
  72. pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
  73. pci_dev_put(dev);
  74. return 0;
  75. }
  76. }
  77. /*
  78. * Intel 450NX errata # 23. Non ascending cacheline evictions to
  79. * write combining memory may resulting in data corruption
  80. */
  81. if (dev->vendor == PCI_VENDOR_ID_INTEL &&
  82. dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
  83. pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
  84. pci_dev_put(dev);
  85. return 0;
  86. }
  87. pci_dev_put(dev);
  88. }
  89. return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
  90. }
  91. /* This function returns the number of variable MTRRs */
  92. static void __init set_num_var_ranges(void)
  93. {
  94. unsigned long config = 0, dummy;
  95. if (use_intel())
  96. rdmsr(MSR_MTRRcap, config, dummy);
  97. else if (is_cpu(AMD))
  98. config = 2;
  99. else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
  100. config = 8;
  101. num_var_ranges = config & 0xff;
  102. }
  103. static void __init init_table(void)
  104. {
  105. int i, max;
  106. max = num_var_ranges;
  107. for (i = 0; i < max; i++)
  108. mtrr_usage_table[i] = 1;
  109. }
  110. struct set_mtrr_data {
  111. atomic_t count;
  112. atomic_t gate;
  113. unsigned long smp_base;
  114. unsigned long smp_size;
  115. unsigned int smp_reg;
  116. mtrr_type smp_type;
  117. };
  118. /**
  119. * ipi_handler - Synchronisation handler. Executed by "other" CPUs.
  120. *
  121. * Returns nothing.
  122. */
  123. static void ipi_handler(void *info)
  124. {
  125. #ifdef CONFIG_SMP
  126. struct set_mtrr_data *data = info;
  127. unsigned long flags;
  128. local_irq_save(flags);
  129. atomic_dec(&data->count);
  130. while (!atomic_read(&data->gate))
  131. cpu_relax();
  132. /* The master has cleared me to execute */
  133. if (data->smp_reg != ~0U) {
  134. mtrr_if->set(data->smp_reg, data->smp_base,
  135. data->smp_size, data->smp_type);
  136. } else if (mtrr_aps_delayed_init) {
  137. /*
  138. * Initialize the MTRRs inaddition to the synchronisation.
  139. */
  140. mtrr_if->set_all();
  141. }
  142. atomic_dec(&data->count);
  143. while (atomic_read(&data->gate))
  144. cpu_relax();
  145. atomic_dec(&data->count);
  146. local_irq_restore(flags);
  147. #endif
  148. }
  149. static inline int types_compatible(mtrr_type type1, mtrr_type type2)
  150. {
  151. return type1 == MTRR_TYPE_UNCACHABLE ||
  152. type2 == MTRR_TYPE_UNCACHABLE ||
  153. (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
  154. (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
  155. }
  156. /**
  157. * set_mtrr - update mtrrs on all processors
  158. * @reg: mtrr in question
  159. * @base: mtrr base
  160. * @size: mtrr size
  161. * @type: mtrr type
  162. *
  163. * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
  164. *
  165. * 1. Send IPI to do the following:
  166. * 2. Disable Interrupts
  167. * 3. Wait for all procs to do so
  168. * 4. Enter no-fill cache mode
  169. * 5. Flush caches
  170. * 6. Clear PGE bit
  171. * 7. Flush all TLBs
  172. * 8. Disable all range registers
  173. * 9. Update the MTRRs
  174. * 10. Enable all range registers
  175. * 11. Flush all TLBs and caches again
  176. * 12. Enter normal cache mode and reenable caching
  177. * 13. Set PGE
  178. * 14. Wait for buddies to catch up
  179. * 15. Enable interrupts.
  180. *
  181. * What does that mean for us? Well, first we set data.count to the number
  182. * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait
  183. * until it hits 0 and proceed. We set the data.gate flag and reset data.count.
  184. * Meanwhile, they are waiting for that flag to be set. Once it's set, each
  185. * CPU goes through the transition of updating MTRRs.
  186. * The CPU vendors may each do it differently,
  187. * so we call mtrr_if->set() callback and let them take care of it.
  188. * When they're done, they again decrement data->count and wait for data.gate
  189. * to be reset.
  190. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag
  191. * Everyone then enables interrupts and we all continue on.
  192. *
  193. * Note that the mechanism is the same for UP systems, too; all the SMP stuff
  194. * becomes nops.
  195. */
  196. static void
  197. set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
  198. {
  199. struct set_mtrr_data data;
  200. unsigned long flags;
  201. data.smp_reg = reg;
  202. data.smp_base = base;
  203. data.smp_size = size;
  204. data.smp_type = type;
  205. atomic_set(&data.count, num_booting_cpus() - 1);
  206. /* Make sure data.count is visible before unleashing other CPUs */
  207. smp_wmb();
  208. atomic_set(&data.gate, 0);
  209. /* Start the ball rolling on other CPUs */
  210. if (smp_call_function(ipi_handler, &data, 0) != 0)
  211. panic("mtrr: timed out waiting for other CPUs\n");
  212. local_irq_save(flags);
  213. while (atomic_read(&data.count))
  214. cpu_relax();
  215. /* Ok, reset count and toggle gate */
  216. atomic_set(&data.count, num_booting_cpus() - 1);
  217. smp_wmb();
  218. atomic_set(&data.gate, 1);
  219. /* Do our MTRR business */
  220. /*
  221. * HACK!
  222. * We use this same function to initialize the mtrrs on boot.
  223. * The state of the boot cpu's mtrrs has been saved, and we want
  224. * to replicate across all the APs.
  225. * If we're doing that @reg is set to something special...
  226. */
  227. if (reg != ~0U)
  228. mtrr_if->set(reg, base, size, type);
  229. else if (!mtrr_aps_delayed_init)
  230. mtrr_if->set_all();
  231. /* Wait for the others */
  232. while (atomic_read(&data.count))
  233. cpu_relax();
  234. atomic_set(&data.count, num_booting_cpus() - 1);
  235. smp_wmb();
  236. atomic_set(&data.gate, 0);
  237. /*
  238. * Wait here for everyone to have seen the gate change
  239. * So we're the last ones to touch 'data'
  240. */
  241. while (atomic_read(&data.count))
  242. cpu_relax();
  243. local_irq_restore(flags);
  244. }
  245. /**
  246. * mtrr_add_page - Add a memory type region
  247. * @base: Physical base address of region in pages (in units of 4 kB!)
  248. * @size: Physical size of region in pages (4 kB)
  249. * @type: Type of MTRR desired
  250. * @increment: If this is true do usage counting on the region
  251. *
  252. * Memory type region registers control the caching on newer Intel and
  253. * non Intel processors. This function allows drivers to request an
  254. * MTRR is added. The details and hardware specifics of each processor's
  255. * implementation are hidden from the caller, but nevertheless the
  256. * caller should expect to need to provide a power of two size on an
  257. * equivalent power of two boundary.
  258. *
  259. * If the region cannot be added either because all regions are in use
  260. * or the CPU cannot support it a negative value is returned. On success
  261. * the register number for this entry is returned, but should be treated
  262. * as a cookie only.
  263. *
  264. * On a multiprocessor machine the changes are made to all processors.
  265. * This is required on x86 by the Intel processors.
  266. *
  267. * The available types are
  268. *
  269. * %MTRR_TYPE_UNCACHABLE - No caching
  270. *
  271. * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  272. *
  273. * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  274. *
  275. * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  276. *
  277. * BUGS: Needs a quiet flag for the cases where drivers do not mind
  278. * failures and do not wish system log messages to be sent.
  279. */
  280. int mtrr_add_page(unsigned long base, unsigned long size,
  281. unsigned int type, bool increment)
  282. {
  283. unsigned long lbase, lsize;
  284. int i, replace, error;
  285. mtrr_type ltype;
  286. if (!mtrr_if)
  287. return -ENXIO;
  288. error = mtrr_if->validate_add_page(base, size, type);
  289. if (error)
  290. return error;
  291. if (type >= MTRR_NUM_TYPES) {
  292. pr_warning("mtrr: type: %u invalid\n", type);
  293. return -EINVAL;
  294. }
  295. /* If the type is WC, check that this processor supports it */
  296. if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
  297. pr_warning("mtrr: your processor doesn't support write-combining\n");
  298. return -ENOSYS;
  299. }
  300. if (!size) {
  301. pr_warning("mtrr: zero sized request\n");
  302. return -EINVAL;
  303. }
  304. if (base & size_or_mask || size & size_or_mask) {
  305. pr_warning("mtrr: base or size exceeds the MTRR width\n");
  306. return -EINVAL;
  307. }
  308. error = -EINVAL;
  309. replace = -1;
  310. /* No CPU hotplug when we change MTRR entries */
  311. get_online_cpus();
  312. /* Search for existing MTRR */
  313. mutex_lock(&mtrr_mutex);
  314. for (i = 0; i < num_var_ranges; ++i) {
  315. mtrr_if->get(i, &lbase, &lsize, &ltype);
  316. if (!lsize || base > lbase + lsize - 1 ||
  317. base + size - 1 < lbase)
  318. continue;
  319. /*
  320. * At this point we know there is some kind of
  321. * overlap/enclosure
  322. */
  323. if (base < lbase || base + size - 1 > lbase + lsize - 1) {
  324. if (base <= lbase &&
  325. base + size - 1 >= lbase + lsize - 1) {
  326. /* New region encloses an existing region */
  327. if (type == ltype) {
  328. replace = replace == -1 ? i : -2;
  329. continue;
  330. } else if (types_compatible(type, ltype))
  331. continue;
  332. }
  333. pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
  334. " 0x%lx000,0x%lx000\n", base, size, lbase,
  335. lsize);
  336. goto out;
  337. }
  338. /* New region is enclosed by an existing region */
  339. if (ltype != type) {
  340. if (types_compatible(type, ltype))
  341. continue;
  342. pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
  343. base, size, mtrr_attrib_to_str(ltype),
  344. mtrr_attrib_to_str(type));
  345. goto out;
  346. }
  347. if (increment)
  348. ++mtrr_usage_table[i];
  349. error = i;
  350. goto out;
  351. }
  352. /* Search for an empty MTRR */
  353. i = mtrr_if->get_free_region(base, size, replace);
  354. if (i >= 0) {
  355. set_mtrr(i, base, size, type);
  356. if (likely(replace < 0)) {
  357. mtrr_usage_table[i] = 1;
  358. } else {
  359. mtrr_usage_table[i] = mtrr_usage_table[replace];
  360. if (increment)
  361. mtrr_usage_table[i]++;
  362. if (unlikely(replace != i)) {
  363. set_mtrr(replace, 0, 0, 0);
  364. mtrr_usage_table[replace] = 0;
  365. }
  366. }
  367. } else {
  368. pr_info("mtrr: no more MTRRs available\n");
  369. }
  370. error = i;
  371. out:
  372. mutex_unlock(&mtrr_mutex);
  373. put_online_cpus();
  374. return error;
  375. }
  376. static int mtrr_check(unsigned long base, unsigned long size)
  377. {
  378. if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
  379. pr_warning("mtrr: size and base must be multiples of 4 kiB\n");
  380. pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base);
  381. dump_stack();
  382. return -1;
  383. }
  384. return 0;
  385. }
  386. /**
  387. * mtrr_add - Add a memory type region
  388. * @base: Physical base address of region
  389. * @size: Physical size of region
  390. * @type: Type of MTRR desired
  391. * @increment: If this is true do usage counting on the region
  392. *
  393. * Memory type region registers control the caching on newer Intel and
  394. * non Intel processors. This function allows drivers to request an
  395. * MTRR is added. The details and hardware specifics of each processor's
  396. * implementation are hidden from the caller, but nevertheless the
  397. * caller should expect to need to provide a power of two size on an
  398. * equivalent power of two boundary.
  399. *
  400. * If the region cannot be added either because all regions are in use
  401. * or the CPU cannot support it a negative value is returned. On success
  402. * the register number for this entry is returned, but should be treated
  403. * as a cookie only.
  404. *
  405. * On a multiprocessor machine the changes are made to all processors.
  406. * This is required on x86 by the Intel processors.
  407. *
  408. * The available types are
  409. *
  410. * %MTRR_TYPE_UNCACHABLE - No caching
  411. *
  412. * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  413. *
  414. * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  415. *
  416. * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  417. *
  418. * BUGS: Needs a quiet flag for the cases where drivers do not mind
  419. * failures and do not wish system log messages to be sent.
  420. */
  421. int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
  422. bool increment)
  423. {
  424. if (mtrr_check(base, size))
  425. return -EINVAL;
  426. return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
  427. increment);
  428. }
  429. EXPORT_SYMBOL(mtrr_add);
  430. /**
  431. * mtrr_del_page - delete a memory type region
  432. * @reg: Register returned by mtrr_add
  433. * @base: Physical base address
  434. * @size: Size of region
  435. *
  436. * If register is supplied then base and size are ignored. This is
  437. * how drivers should call it.
  438. *
  439. * Releases an MTRR region. If the usage count drops to zero the
  440. * register is freed and the region returns to default state.
  441. * On success the register is returned, on failure a negative error
  442. * code.
  443. */
  444. int mtrr_del_page(int reg, unsigned long base, unsigned long size)
  445. {
  446. int i, max;
  447. mtrr_type ltype;
  448. unsigned long lbase, lsize;
  449. int error = -EINVAL;
  450. if (!mtrr_if)
  451. return -ENXIO;
  452. max = num_var_ranges;
  453. /* No CPU hotplug when we change MTRR entries */
  454. get_online_cpus();
  455. mutex_lock(&mtrr_mutex);
  456. if (reg < 0) {
  457. /* Search for existing MTRR */
  458. for (i = 0; i < max; ++i) {
  459. mtrr_if->get(i, &lbase, &lsize, &ltype);
  460. if (lbase == base && lsize == size) {
  461. reg = i;
  462. break;
  463. }
  464. }
  465. if (reg < 0) {
  466. pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
  467. base, size);
  468. goto out;
  469. }
  470. }
  471. if (reg >= max) {
  472. pr_warning("mtrr: register: %d too big\n", reg);
  473. goto out;
  474. }
  475. mtrr_if->get(reg, &lbase, &lsize, &ltype);
  476. if (lsize < 1) {
  477. pr_warning("mtrr: MTRR %d not used\n", reg);
  478. goto out;
  479. }
  480. if (mtrr_usage_table[reg] < 1) {
  481. pr_warning("mtrr: reg: %d has count=0\n", reg);
  482. goto out;
  483. }
  484. if (--mtrr_usage_table[reg] < 1)
  485. set_mtrr(reg, 0, 0, 0);
  486. error = reg;
  487. out:
  488. mutex_unlock(&mtrr_mutex);
  489. put_online_cpus();
  490. return error;
  491. }
  492. /**
  493. * mtrr_del - delete a memory type region
  494. * @reg: Register returned by mtrr_add
  495. * @base: Physical base address
  496. * @size: Size of region
  497. *
  498. * If register is supplied then base and size are ignored. This is
  499. * how drivers should call it.
  500. *
  501. * Releases an MTRR region. If the usage count drops to zero the
  502. * register is freed and the region returns to default state.
  503. * On success the register is returned, on failure a negative error
  504. * code.
  505. */
  506. int mtrr_del(int reg, unsigned long base, unsigned long size)
  507. {
  508. if (mtrr_check(base, size))
  509. return -EINVAL;
  510. return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
  511. }
  512. EXPORT_SYMBOL(mtrr_del);
  513. /*
  514. * HACK ALERT!
  515. * These should be called implicitly, but we can't yet until all the initcall
  516. * stuff is done...
  517. */
  518. static void __init init_ifs(void)
  519. {
  520. #ifndef CONFIG_X86_64
  521. amd_init_mtrr();
  522. cyrix_init_mtrr();
  523. centaur_init_mtrr();
  524. #endif
  525. }
  526. /* The suspend/resume methods are only for CPU without MTRR. CPU using generic
  527. * MTRR driver doesn't require this
  528. */
  529. struct mtrr_value {
  530. mtrr_type ltype;
  531. unsigned long lbase;
  532. unsigned long lsize;
  533. };
  534. static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
  535. static int mtrr_save(struct sys_device *sysdev, pm_message_t state)
  536. {
  537. int i;
  538. for (i = 0; i < num_var_ranges; i++) {
  539. mtrr_if->get(i, &mtrr_value[i].lbase,
  540. &mtrr_value[i].lsize,
  541. &mtrr_value[i].ltype);
  542. }
  543. return 0;
  544. }
  545. static int mtrr_restore(struct sys_device *sysdev)
  546. {
  547. int i;
  548. for (i = 0; i < num_var_ranges; i++) {
  549. if (mtrr_value[i].lsize) {
  550. set_mtrr(i, mtrr_value[i].lbase,
  551. mtrr_value[i].lsize,
  552. mtrr_value[i].ltype);
  553. }
  554. }
  555. return 0;
  556. }
  557. static struct sysdev_driver mtrr_sysdev_driver = {
  558. .suspend = mtrr_save,
  559. .resume = mtrr_restore,
  560. };
  561. int __initdata changed_by_mtrr_cleanup;
  562. /**
  563. * mtrr_bp_init - initialize mtrrs on the boot CPU
  564. *
  565. * This needs to be called early; before any of the other CPUs are
  566. * initialized (i.e. before smp_init()).
  567. *
  568. */
  569. void __init mtrr_bp_init(void)
  570. {
  571. u32 phys_addr;
  572. init_ifs();
  573. phys_addr = 32;
  574. if (cpu_has_mtrr) {
  575. mtrr_if = &generic_mtrr_ops;
  576. size_or_mask = 0xff000000; /* 36 bits */
  577. size_and_mask = 0x00f00000;
  578. phys_addr = 36;
  579. /*
  580. * This is an AMD specific MSR, but we assume(hope?) that
  581. * Intel will implement it to when they extend the address
  582. * bus of the Xeon.
  583. */
  584. if (cpuid_eax(0x80000000) >= 0x80000008) {
  585. phys_addr = cpuid_eax(0x80000008) & 0xff;
  586. /* CPUID workaround for Intel 0F33/0F34 CPU */
  587. if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
  588. boot_cpu_data.x86 == 0xF &&
  589. boot_cpu_data.x86_model == 0x3 &&
  590. (boot_cpu_data.x86_mask == 0x3 ||
  591. boot_cpu_data.x86_mask == 0x4))
  592. phys_addr = 36;
  593. size_or_mask = ~((1ULL << (phys_addr - PAGE_SHIFT)) - 1);
  594. size_and_mask = ~size_or_mask & 0xfffff00000ULL;
  595. } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
  596. boot_cpu_data.x86 == 6) {
  597. /*
  598. * VIA C* family have Intel style MTRRs,
  599. * but don't support PAE
  600. */
  601. size_or_mask = 0xfff00000; /* 32 bits */
  602. size_and_mask = 0;
  603. phys_addr = 32;
  604. }
  605. } else {
  606. switch (boot_cpu_data.x86_vendor) {
  607. case X86_VENDOR_AMD:
  608. if (cpu_has_k6_mtrr) {
  609. /* Pre-Athlon (K6) AMD CPU MTRRs */
  610. mtrr_if = mtrr_ops[X86_VENDOR_AMD];
  611. size_or_mask = 0xfff00000; /* 32 bits */
  612. size_and_mask = 0;
  613. }
  614. break;
  615. case X86_VENDOR_CENTAUR:
  616. if (cpu_has_centaur_mcr) {
  617. mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
  618. size_or_mask = 0xfff00000; /* 32 bits */
  619. size_and_mask = 0;
  620. }
  621. break;
  622. case X86_VENDOR_CYRIX:
  623. if (cpu_has_cyrix_arr) {
  624. mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
  625. size_or_mask = 0xfff00000; /* 32 bits */
  626. size_and_mask = 0;
  627. }
  628. break;
  629. default:
  630. break;
  631. }
  632. }
  633. if (mtrr_if) {
  634. set_num_var_ranges();
  635. init_table();
  636. if (use_intel()) {
  637. get_mtrr_state();
  638. if (mtrr_cleanup(phys_addr)) {
  639. changed_by_mtrr_cleanup = 1;
  640. mtrr_if->set_all();
  641. }
  642. }
  643. }
  644. }
  645. void mtrr_ap_init(void)
  646. {
  647. if (!use_intel() || mtrr_aps_delayed_init)
  648. return;
  649. /*
  650. * Ideally we should hold mtrr_mutex here to avoid mtrr entries
  651. * changed, but this routine will be called in cpu boot time,
  652. * holding the lock breaks it.
  653. *
  654. * This routine is called in two cases:
  655. *
  656. * 1. very earily time of software resume, when there absolutely
  657. * isn't mtrr entry changes;
  658. *
  659. * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
  660. * lock to prevent mtrr entry changes
  661. */
  662. set_mtrr(~0U, 0, 0, 0);
  663. }
  664. /**
  665. * Save current fixed-range MTRR state of the BSP
  666. */
  667. void mtrr_save_state(void)
  668. {
  669. smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
  670. }
  671. void set_mtrr_aps_delayed_init(void)
  672. {
  673. if (!use_intel())
  674. return;
  675. mtrr_aps_delayed_init = true;
  676. }
  677. /*
  678. * MTRR initialization for all AP's
  679. */
  680. void mtrr_aps_init(void)
  681. {
  682. if (!use_intel())
  683. return;
  684. set_mtrr(~0U, 0, 0, 0);
  685. mtrr_aps_delayed_init = false;
  686. }
  687. void mtrr_bp_restore(void)
  688. {
  689. if (!use_intel())
  690. return;
  691. mtrr_if->set_all();
  692. }
  693. static int __init mtrr_init_finialize(void)
  694. {
  695. if (!mtrr_if)
  696. return 0;
  697. if (use_intel()) {
  698. if (!changed_by_mtrr_cleanup)
  699. mtrr_state_warn();
  700. return 0;
  701. }
  702. /*
  703. * The CPU has no MTRR and seems to not support SMP. They have
  704. * specific drivers, we use a tricky method to support
  705. * suspend/resume for them.
  706. *
  707. * TBD: is there any system with such CPU which supports
  708. * suspend/resume? If no, we should remove the code.
  709. */
  710. sysdev_driver_register(&cpu_sysdev_class, &mtrr_sysdev_driver);
  711. return 0;
  712. }
  713. subsys_initcall(mtrr_init_finialize);