main.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808
  1. /* Generic MTRR (Memory Type Range Register) driver.
  2. Copyright (C) 1997-2000 Richard Gooch
  3. Copyright (c) 2002 Patrick Mochel
  4. This library is free software; you can redistribute it and/or
  5. modify it under the terms of the GNU Library General Public
  6. License as published by the Free Software Foundation; either
  7. version 2 of the License, or (at your option) any later version.
  8. This library is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. Library General Public License for more details.
  12. You should have received a copy of the GNU Library General Public
  13. License along with this library; if not, write to the Free
  14. Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  15. Richard Gooch may be reached by email at rgooch@atnf.csiro.au
  16. The postal address is:
  17. Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
  18. Source: "Pentium Pro Family Developer's Manual, Volume 3:
  19. Operating System Writer's Guide" (Intel document number 242692),
  20. section 11.11.7
  21. This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
  22. on 6-7 March 2002.
  23. Source: Intel Architecture Software Developers Manual, Volume 3:
  24. System Programming Guide; Section 9.11. (1997 edition - PPro).
  25. */
  26. #define DEBUG
  27. #include <linux/types.h> /* FIXME: kvm_para.h needs this */
  28. #include <linux/kvm_para.h>
  29. #include <linux/uaccess.h>
  30. #include <linux/module.h>
  31. #include <linux/mutex.h>
  32. #include <linux/init.h>
  33. #include <linux/sort.h>
  34. #include <linux/cpu.h>
  35. #include <linux/pci.h>
  36. #include <linux/smp.h>
  37. #include <asm/processor.h>
  38. #include <asm/e820.h>
  39. #include <asm/mtrr.h>
  40. #include <asm/msr.h>
  41. #include "mtrr.h"
  42. u32 num_var_ranges;
  43. unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
  44. static DEFINE_MUTEX(mtrr_mutex);
  45. u64 size_or_mask, size_and_mask;
  46. static bool mtrr_aps_delayed_init;
  47. static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
  48. const struct mtrr_ops *mtrr_if;
  49. static void set_mtrr(unsigned int reg, unsigned long base,
  50. unsigned long size, mtrr_type type);
  51. void set_mtrr_ops(const struct mtrr_ops *ops)
  52. {
  53. if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
  54. mtrr_ops[ops->vendor] = ops;
  55. }
  56. /* Returns non-zero if we have the write-combining memory type */
  57. static int have_wrcomb(void)
  58. {
  59. struct pci_dev *dev;
  60. u8 rev;
  61. dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
  62. if (dev != NULL) {
  63. /*
  64. * ServerWorks LE chipsets < rev 6 have problems with
  65. * write-combining. Don't allow it and leave room for other
  66. * chipsets to be tagged
  67. */
  68. if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
  69. dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) {
  70. pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev);
  71. if (rev <= 5) {
  72. pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
  73. pci_dev_put(dev);
  74. return 0;
  75. }
  76. }
  77. /*
  78. * Intel 450NX errata # 23. Non ascending cacheline evictions to
  79. * write combining memory may resulting in data corruption
  80. */
  81. if (dev->vendor == PCI_VENDOR_ID_INTEL &&
  82. dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
  83. pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
  84. pci_dev_put(dev);
  85. return 0;
  86. }
  87. pci_dev_put(dev);
  88. }
  89. return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
  90. }
  91. /* This function returns the number of variable MTRRs */
  92. static void __init set_num_var_ranges(void)
  93. {
  94. unsigned long config = 0, dummy;
  95. if (use_intel())
  96. rdmsr(MSR_MTRRcap, config, dummy);
  97. else if (is_cpu(AMD))
  98. config = 2;
  99. else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
  100. config = 8;
  101. num_var_ranges = config & 0xff;
  102. }
  103. static void __init init_table(void)
  104. {
  105. int i, max;
  106. max = num_var_ranges;
  107. for (i = 0; i < max; i++)
  108. mtrr_usage_table[i] = 1;
  109. }
  110. struct set_mtrr_data {
  111. atomic_t count;
  112. atomic_t gate;
  113. unsigned long smp_base;
  114. unsigned long smp_size;
  115. unsigned int smp_reg;
  116. mtrr_type smp_type;
  117. };
  118. /**
  119. * ipi_handler - Synchronisation handler. Executed by "other" CPUs.
  120. * @info: pointer to mtrr configuration data
  121. *
  122. * Returns nothing.
  123. */
  124. static void ipi_handler(void *info)
  125. {
  126. #ifdef CONFIG_SMP
  127. struct set_mtrr_data *data = info;
  128. unsigned long flags;
  129. local_irq_save(flags);
  130. atomic_dec(&data->count);
  131. while (!atomic_read(&data->gate))
  132. cpu_relax();
  133. /* The master has cleared me to execute */
  134. if (data->smp_reg != ~0U) {
  135. mtrr_if->set(data->smp_reg, data->smp_base,
  136. data->smp_size, data->smp_type);
  137. } else if (mtrr_aps_delayed_init) {
  138. /*
  139. * Initialize the MTRRs inaddition to the synchronisation.
  140. */
  141. mtrr_if->set_all();
  142. }
  143. atomic_dec(&data->count);
  144. while (atomic_read(&data->gate))
  145. cpu_relax();
  146. atomic_dec(&data->count);
  147. local_irq_restore(flags);
  148. #endif
  149. }
  150. static inline int types_compatible(mtrr_type type1, mtrr_type type2)
  151. {
  152. return type1 == MTRR_TYPE_UNCACHABLE ||
  153. type2 == MTRR_TYPE_UNCACHABLE ||
  154. (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
  155. (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
  156. }
  157. /**
  158. * set_mtrr - update mtrrs on all processors
  159. * @reg: mtrr in question
  160. * @base: mtrr base
  161. * @size: mtrr size
  162. * @type: mtrr type
  163. *
  164. * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
  165. *
  166. * 1. Send IPI to do the following:
  167. * 2. Disable Interrupts
  168. * 3. Wait for all procs to do so
  169. * 4. Enter no-fill cache mode
  170. * 5. Flush caches
  171. * 6. Clear PGE bit
  172. * 7. Flush all TLBs
  173. * 8. Disable all range registers
  174. * 9. Update the MTRRs
  175. * 10. Enable all range registers
  176. * 11. Flush all TLBs and caches again
  177. * 12. Enter normal cache mode and reenable caching
  178. * 13. Set PGE
  179. * 14. Wait for buddies to catch up
  180. * 15. Enable interrupts.
  181. *
  182. * What does that mean for us? Well, first we set data.count to the number
  183. * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait
  184. * until it hits 0 and proceed. We set the data.gate flag and reset data.count.
  185. * Meanwhile, they are waiting for that flag to be set. Once it's set, each
  186. * CPU goes through the transition of updating MTRRs.
  187. * The CPU vendors may each do it differently,
  188. * so we call mtrr_if->set() callback and let them take care of it.
  189. * When they're done, they again decrement data->count and wait for data.gate
  190. * to be reset.
  191. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag
  192. * Everyone then enables interrupts and we all continue on.
  193. *
  194. * Note that the mechanism is the same for UP systems, too; all the SMP stuff
  195. * becomes nops.
  196. */
  197. static void
  198. set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
  199. {
  200. struct set_mtrr_data data;
  201. unsigned long flags;
  202. data.smp_reg = reg;
  203. data.smp_base = base;
  204. data.smp_size = size;
  205. data.smp_type = type;
  206. atomic_set(&data.count, num_booting_cpus() - 1);
  207. /* Make sure data.count is visible before unleashing other CPUs */
  208. smp_wmb();
  209. atomic_set(&data.gate, 0);
  210. /* Start the ball rolling on other CPUs */
  211. if (smp_call_function(ipi_handler, &data, 0) != 0)
  212. panic("mtrr: timed out waiting for other CPUs\n");
  213. local_irq_save(flags);
  214. while (atomic_read(&data.count))
  215. cpu_relax();
  216. /* Ok, reset count and toggle gate */
  217. atomic_set(&data.count, num_booting_cpus() - 1);
  218. smp_wmb();
  219. atomic_set(&data.gate, 1);
  220. /* Do our MTRR business */
  221. /*
  222. * HACK!
  223. * We use this same function to initialize the mtrrs on boot.
  224. * The state of the boot cpu's mtrrs has been saved, and we want
  225. * to replicate across all the APs.
  226. * If we're doing that @reg is set to something special...
  227. */
  228. if (reg != ~0U)
  229. mtrr_if->set(reg, base, size, type);
  230. else if (!mtrr_aps_delayed_init)
  231. mtrr_if->set_all();
  232. /* Wait for the others */
  233. while (atomic_read(&data.count))
  234. cpu_relax();
  235. atomic_set(&data.count, num_booting_cpus() - 1);
  236. smp_wmb();
  237. atomic_set(&data.gate, 0);
  238. /*
  239. * Wait here for everyone to have seen the gate change
  240. * So we're the last ones to touch 'data'
  241. */
  242. while (atomic_read(&data.count))
  243. cpu_relax();
  244. local_irq_restore(flags);
  245. }
  246. /**
  247. * mtrr_add_page - Add a memory type region
  248. * @base: Physical base address of region in pages (in units of 4 kB!)
  249. * @size: Physical size of region in pages (4 kB)
  250. * @type: Type of MTRR desired
  251. * @increment: If this is true do usage counting on the region
  252. *
  253. * Memory type region registers control the caching on newer Intel and
  254. * non Intel processors. This function allows drivers to request an
  255. * MTRR is added. The details and hardware specifics of each processor's
  256. * implementation are hidden from the caller, but nevertheless the
  257. * caller should expect to need to provide a power of two size on an
  258. * equivalent power of two boundary.
  259. *
  260. * If the region cannot be added either because all regions are in use
  261. * or the CPU cannot support it a negative value is returned. On success
  262. * the register number for this entry is returned, but should be treated
  263. * as a cookie only.
  264. *
  265. * On a multiprocessor machine the changes are made to all processors.
  266. * This is required on x86 by the Intel processors.
  267. *
  268. * The available types are
  269. *
  270. * %MTRR_TYPE_UNCACHABLE - No caching
  271. *
  272. * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  273. *
  274. * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  275. *
  276. * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  277. *
  278. * BUGS: Needs a quiet flag for the cases where drivers do not mind
  279. * failures and do not wish system log messages to be sent.
  280. */
  281. int mtrr_add_page(unsigned long base, unsigned long size,
  282. unsigned int type, bool increment)
  283. {
  284. unsigned long lbase, lsize;
  285. int i, replace, error;
  286. mtrr_type ltype;
  287. if (!mtrr_if)
  288. return -ENXIO;
  289. error = mtrr_if->validate_add_page(base, size, type);
  290. if (error)
  291. return error;
  292. if (type >= MTRR_NUM_TYPES) {
  293. pr_warning("mtrr: type: %u invalid\n", type);
  294. return -EINVAL;
  295. }
  296. /* If the type is WC, check that this processor supports it */
  297. if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
  298. pr_warning("mtrr: your processor doesn't support write-combining\n");
  299. return -ENOSYS;
  300. }
  301. if (!size) {
  302. pr_warning("mtrr: zero sized request\n");
  303. return -EINVAL;
  304. }
  305. if (base & size_or_mask || size & size_or_mask) {
  306. pr_warning("mtrr: base or size exceeds the MTRR width\n");
  307. return -EINVAL;
  308. }
  309. error = -EINVAL;
  310. replace = -1;
  311. /* No CPU hotplug when we change MTRR entries */
  312. get_online_cpus();
  313. /* Search for existing MTRR */
  314. mutex_lock(&mtrr_mutex);
  315. for (i = 0; i < num_var_ranges; ++i) {
  316. mtrr_if->get(i, &lbase, &lsize, &ltype);
  317. if (!lsize || base > lbase + lsize - 1 ||
  318. base + size - 1 < lbase)
  319. continue;
  320. /*
  321. * At this point we know there is some kind of
  322. * overlap/enclosure
  323. */
  324. if (base < lbase || base + size - 1 > lbase + lsize - 1) {
  325. if (base <= lbase &&
  326. base + size - 1 >= lbase + lsize - 1) {
  327. /* New region encloses an existing region */
  328. if (type == ltype) {
  329. replace = replace == -1 ? i : -2;
  330. continue;
  331. } else if (types_compatible(type, ltype))
  332. continue;
  333. }
  334. pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
  335. " 0x%lx000,0x%lx000\n", base, size, lbase,
  336. lsize);
  337. goto out;
  338. }
  339. /* New region is enclosed by an existing region */
  340. if (ltype != type) {
  341. if (types_compatible(type, ltype))
  342. continue;
  343. pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
  344. base, size, mtrr_attrib_to_str(ltype),
  345. mtrr_attrib_to_str(type));
  346. goto out;
  347. }
  348. if (increment)
  349. ++mtrr_usage_table[i];
  350. error = i;
  351. goto out;
  352. }
  353. /* Search for an empty MTRR */
  354. i = mtrr_if->get_free_region(base, size, replace);
  355. if (i >= 0) {
  356. set_mtrr(i, base, size, type);
  357. if (likely(replace < 0)) {
  358. mtrr_usage_table[i] = 1;
  359. } else {
  360. mtrr_usage_table[i] = mtrr_usage_table[replace];
  361. if (increment)
  362. mtrr_usage_table[i]++;
  363. if (unlikely(replace != i)) {
  364. set_mtrr(replace, 0, 0, 0);
  365. mtrr_usage_table[replace] = 0;
  366. }
  367. }
  368. } else {
  369. pr_info("mtrr: no more MTRRs available\n");
  370. }
  371. error = i;
  372. out:
  373. mutex_unlock(&mtrr_mutex);
  374. put_online_cpus();
  375. return error;
  376. }
  377. static int mtrr_check(unsigned long base, unsigned long size)
  378. {
  379. if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
  380. pr_warning("mtrr: size and base must be multiples of 4 kiB\n");
  381. pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base);
  382. dump_stack();
  383. return -1;
  384. }
  385. return 0;
  386. }
  387. /**
  388. * mtrr_add - Add a memory type region
  389. * @base: Physical base address of region
  390. * @size: Physical size of region
  391. * @type: Type of MTRR desired
  392. * @increment: If this is true do usage counting on the region
  393. *
  394. * Memory type region registers control the caching on newer Intel and
  395. * non Intel processors. This function allows drivers to request an
  396. * MTRR is added. The details and hardware specifics of each processor's
  397. * implementation are hidden from the caller, but nevertheless the
  398. * caller should expect to need to provide a power of two size on an
  399. * equivalent power of two boundary.
  400. *
  401. * If the region cannot be added either because all regions are in use
  402. * or the CPU cannot support it a negative value is returned. On success
  403. * the register number for this entry is returned, but should be treated
  404. * as a cookie only.
  405. *
  406. * On a multiprocessor machine the changes are made to all processors.
  407. * This is required on x86 by the Intel processors.
  408. *
  409. * The available types are
  410. *
  411. * %MTRR_TYPE_UNCACHABLE - No caching
  412. *
  413. * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  414. *
  415. * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  416. *
  417. * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  418. *
  419. * BUGS: Needs a quiet flag for the cases where drivers do not mind
  420. * failures and do not wish system log messages to be sent.
  421. */
  422. int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
  423. bool increment)
  424. {
  425. if (mtrr_check(base, size))
  426. return -EINVAL;
  427. return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
  428. increment);
  429. }
  430. EXPORT_SYMBOL(mtrr_add);
  431. /**
  432. * mtrr_del_page - delete a memory type region
  433. * @reg: Register returned by mtrr_add
  434. * @base: Physical base address
  435. * @size: Size of region
  436. *
  437. * If register is supplied then base and size are ignored. This is
  438. * how drivers should call it.
  439. *
  440. * Releases an MTRR region. If the usage count drops to zero the
  441. * register is freed and the region returns to default state.
  442. * On success the register is returned, on failure a negative error
  443. * code.
  444. */
  445. int mtrr_del_page(int reg, unsigned long base, unsigned long size)
  446. {
  447. int i, max;
  448. mtrr_type ltype;
  449. unsigned long lbase, lsize;
  450. int error = -EINVAL;
  451. if (!mtrr_if)
  452. return -ENXIO;
  453. max = num_var_ranges;
  454. /* No CPU hotplug when we change MTRR entries */
  455. get_online_cpus();
  456. mutex_lock(&mtrr_mutex);
  457. if (reg < 0) {
  458. /* Search for existing MTRR */
  459. for (i = 0; i < max; ++i) {
  460. mtrr_if->get(i, &lbase, &lsize, &ltype);
  461. if (lbase == base && lsize == size) {
  462. reg = i;
  463. break;
  464. }
  465. }
  466. if (reg < 0) {
  467. pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
  468. base, size);
  469. goto out;
  470. }
  471. }
  472. if (reg >= max) {
  473. pr_warning("mtrr: register: %d too big\n", reg);
  474. goto out;
  475. }
  476. mtrr_if->get(reg, &lbase, &lsize, &ltype);
  477. if (lsize < 1) {
  478. pr_warning("mtrr: MTRR %d not used\n", reg);
  479. goto out;
  480. }
  481. if (mtrr_usage_table[reg] < 1) {
  482. pr_warning("mtrr: reg: %d has count=0\n", reg);
  483. goto out;
  484. }
  485. if (--mtrr_usage_table[reg] < 1)
  486. set_mtrr(reg, 0, 0, 0);
  487. error = reg;
  488. out:
  489. mutex_unlock(&mtrr_mutex);
  490. put_online_cpus();
  491. return error;
  492. }
  493. /**
  494. * mtrr_del - delete a memory type region
  495. * @reg: Register returned by mtrr_add
  496. * @base: Physical base address
  497. * @size: Size of region
  498. *
  499. * If register is supplied then base and size are ignored. This is
  500. * how drivers should call it.
  501. *
  502. * Releases an MTRR region. If the usage count drops to zero the
  503. * register is freed and the region returns to default state.
  504. * On success the register is returned, on failure a negative error
  505. * code.
  506. */
  507. int mtrr_del(int reg, unsigned long base, unsigned long size)
  508. {
  509. if (mtrr_check(base, size))
  510. return -EINVAL;
  511. return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
  512. }
  513. EXPORT_SYMBOL(mtrr_del);
  514. /*
  515. * HACK ALERT!
  516. * These should be called implicitly, but we can't yet until all the initcall
  517. * stuff is done...
  518. */
  519. static void __init init_ifs(void)
  520. {
  521. #ifndef CONFIG_X86_64
  522. amd_init_mtrr();
  523. cyrix_init_mtrr();
  524. centaur_init_mtrr();
  525. #endif
  526. }
  527. /* The suspend/resume methods are only for CPU without MTRR. CPU using generic
  528. * MTRR driver doesn't require this
  529. */
  530. struct mtrr_value {
  531. mtrr_type ltype;
  532. unsigned long lbase;
  533. unsigned long lsize;
  534. };
  535. static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
  536. static int mtrr_save(struct sys_device *sysdev, pm_message_t state)
  537. {
  538. int i;
  539. for (i = 0; i < num_var_ranges; i++) {
  540. mtrr_if->get(i, &mtrr_value[i].lbase,
  541. &mtrr_value[i].lsize,
  542. &mtrr_value[i].ltype);
  543. }
  544. return 0;
  545. }
  546. static int mtrr_restore(struct sys_device *sysdev)
  547. {
  548. int i;
  549. for (i = 0; i < num_var_ranges; i++) {
  550. if (mtrr_value[i].lsize) {
  551. set_mtrr(i, mtrr_value[i].lbase,
  552. mtrr_value[i].lsize,
  553. mtrr_value[i].ltype);
  554. }
  555. }
  556. return 0;
  557. }
  558. static struct sysdev_driver mtrr_sysdev_driver = {
  559. .suspend = mtrr_save,
  560. .resume = mtrr_restore,
  561. };
  562. int __initdata changed_by_mtrr_cleanup;
  563. /**
  564. * mtrr_bp_init - initialize mtrrs on the boot CPU
  565. *
  566. * This needs to be called early; before any of the other CPUs are
  567. * initialized (i.e. before smp_init()).
  568. *
  569. */
  570. void __init mtrr_bp_init(void)
  571. {
  572. u32 phys_addr;
  573. init_ifs();
  574. phys_addr = 32;
  575. if (cpu_has_mtrr) {
  576. mtrr_if = &generic_mtrr_ops;
  577. size_or_mask = 0xff000000; /* 36 bits */
  578. size_and_mask = 0x00f00000;
  579. phys_addr = 36;
  580. /*
  581. * This is an AMD specific MSR, but we assume(hope?) that
  582. * Intel will implement it to when they extend the address
  583. * bus of the Xeon.
  584. */
  585. if (cpuid_eax(0x80000000) >= 0x80000008) {
  586. phys_addr = cpuid_eax(0x80000008) & 0xff;
  587. /* CPUID workaround for Intel 0F33/0F34 CPU */
  588. if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
  589. boot_cpu_data.x86 == 0xF &&
  590. boot_cpu_data.x86_model == 0x3 &&
  591. (boot_cpu_data.x86_mask == 0x3 ||
  592. boot_cpu_data.x86_mask == 0x4))
  593. phys_addr = 36;
  594. size_or_mask = ~((1ULL << (phys_addr - PAGE_SHIFT)) - 1);
  595. size_and_mask = ~size_or_mask & 0xfffff00000ULL;
  596. } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
  597. boot_cpu_data.x86 == 6) {
  598. /*
  599. * VIA C* family have Intel style MTRRs,
  600. * but don't support PAE
  601. */
  602. size_or_mask = 0xfff00000; /* 32 bits */
  603. size_and_mask = 0;
  604. phys_addr = 32;
  605. }
  606. } else {
  607. switch (boot_cpu_data.x86_vendor) {
  608. case X86_VENDOR_AMD:
  609. if (cpu_has_k6_mtrr) {
  610. /* Pre-Athlon (K6) AMD CPU MTRRs */
  611. mtrr_if = mtrr_ops[X86_VENDOR_AMD];
  612. size_or_mask = 0xfff00000; /* 32 bits */
  613. size_and_mask = 0;
  614. }
  615. break;
  616. case X86_VENDOR_CENTAUR:
  617. if (cpu_has_centaur_mcr) {
  618. mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
  619. size_or_mask = 0xfff00000; /* 32 bits */
  620. size_and_mask = 0;
  621. }
  622. break;
  623. case X86_VENDOR_CYRIX:
  624. if (cpu_has_cyrix_arr) {
  625. mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
  626. size_or_mask = 0xfff00000; /* 32 bits */
  627. size_and_mask = 0;
  628. }
  629. break;
  630. default:
  631. break;
  632. }
  633. }
  634. if (mtrr_if) {
  635. set_num_var_ranges();
  636. init_table();
  637. if (use_intel()) {
  638. get_mtrr_state();
  639. if (mtrr_cleanup(phys_addr)) {
  640. changed_by_mtrr_cleanup = 1;
  641. mtrr_if->set_all();
  642. }
  643. }
  644. }
  645. }
  646. void mtrr_ap_init(void)
  647. {
  648. if (!use_intel() || mtrr_aps_delayed_init)
  649. return;
  650. /*
  651. * Ideally we should hold mtrr_mutex here to avoid mtrr entries
  652. * changed, but this routine will be called in cpu boot time,
  653. * holding the lock breaks it.
  654. *
  655. * This routine is called in two cases:
  656. *
  657. * 1. very earily time of software resume, when there absolutely
  658. * isn't mtrr entry changes;
  659. *
  660. * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
  661. * lock to prevent mtrr entry changes
  662. */
  663. set_mtrr(~0U, 0, 0, 0);
  664. }
  665. /**
  666. * Save current fixed-range MTRR state of the BSP
  667. */
  668. void mtrr_save_state(void)
  669. {
  670. smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
  671. }
  672. void set_mtrr_aps_delayed_init(void)
  673. {
  674. if (!use_intel())
  675. return;
  676. mtrr_aps_delayed_init = true;
  677. }
  678. /*
  679. * MTRR initialization for all AP's
  680. */
  681. void mtrr_aps_init(void)
  682. {
  683. if (!use_intel())
  684. return;
  685. set_mtrr(~0U, 0, 0, 0);
  686. mtrr_aps_delayed_init = false;
  687. }
  688. void mtrr_bp_restore(void)
  689. {
  690. if (!use_intel())
  691. return;
  692. mtrr_if->set_all();
  693. }
  694. static int __init mtrr_init_finialize(void)
  695. {
  696. if (!mtrr_if)
  697. return 0;
  698. if (use_intel()) {
  699. if (!changed_by_mtrr_cleanup)
  700. mtrr_state_warn();
  701. return 0;
  702. }
  703. /*
  704. * The CPU has no MTRR and seems to not support SMP. They have
  705. * specific drivers, we use a tricky method to support
  706. * suspend/resume for them.
  707. *
  708. * TBD: is there any system with such CPU which supports
  709. * suspend/resume? If no, we should remove the code.
  710. */
  711. sysdev_driver_register(&cpu_sysdev_class, &mtrr_sysdev_driver);
  712. return 0;
  713. }
  714. subsys_initcall(mtrr_init_finialize);