mmu.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. /*
  2. * Xen mmu operations
  3. *
  4. * This file contains the various mmu fetch and update operations.
  5. * The most important job they must perform is the mapping between the
  6. * domain's pfn and the overall machine mfns.
  7. *
  8. * Xen allows guests to directly update the pagetable, in a controlled
  9. * fashion. In other words, the guest modifies the same pagetable
  10. * that the CPU actually uses, which eliminates the overhead of having
  11. * a separate shadow pagetable.
  12. *
  13. * In order to allow this, it falls on the guest domain to map its
  14. * notion of a "physical" pfn - which is just a domain-local linear
  15. * address - into a real "machine address" which the CPU's MMU can
  16. * use.
  17. *
  18. * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
  19. * inserted directly into the pagetable. When creating a new
  20. * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
  21. * when reading the content back with __(pgd|pmd|pte)_val, it converts
  22. * the mfn back into a pfn.
  23. *
  24. * The other constraint is that all pages which make up a pagetable
  25. * must be mapped read-only in the guest. This prevents uncontrolled
  26. * guest updates to the pagetable. Xen strictly enforces this, and
  27. * will disallow any pagetable update which will end up mapping a
  28. * pagetable page RW, and will disallow using any writable page as a
  29. * pagetable.
  30. *
  31. * Naively, when loading %cr3 with the base of a new pagetable, Xen
  32. * would need to validate the whole pagetable before going on.
  33. * Naturally, this is quite slow. The solution is to "pin" a
  34. * pagetable, which enforces all the constraints on the pagetable even
  35. * when it is not actively in use. This menas that Xen can be assured
  36. * that it is still valid when you do load it into %cr3, and doesn't
  37. * need to revalidate it.
  38. *
  39. * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  40. */
  41. #include <linux/highmem.h>
  42. #include <linux/bug.h>
  43. #include <linux/sched.h>
  44. #include <asm/pgtable.h>
  45. #include <asm/tlbflush.h>
  46. #include <asm/mmu_context.h>
  47. #include <asm/paravirt.h>
  48. #include <asm/xen/hypercall.h>
  49. #include <asm/xen/hypervisor.h>
  50. #include <xen/page.h>
  51. #include <xen/interface/xen.h>
  52. #include "multicalls.h"
  53. #include "mmu.h"
  54. xmaddr_t arbitrary_virt_to_machine(unsigned long address)
  55. {
  56. pte_t *pte = lookup_address(address);
  57. unsigned offset = address & PAGE_MASK;
  58. BUG_ON(pte == NULL);
  59. return XMADDR((pte_mfn(*pte) << PAGE_SHIFT) + offset);
  60. }
  61. void make_lowmem_page_readonly(void *vaddr)
  62. {
  63. pte_t *pte, ptev;
  64. unsigned long address = (unsigned long)vaddr;
  65. pte = lookup_address(address);
  66. BUG_ON(pte == NULL);
  67. ptev = pte_wrprotect(*pte);
  68. if (HYPERVISOR_update_va_mapping(address, ptev, 0))
  69. BUG();
  70. }
  71. void make_lowmem_page_readwrite(void *vaddr)
  72. {
  73. pte_t *pte, ptev;
  74. unsigned long address = (unsigned long)vaddr;
  75. pte = lookup_address(address);
  76. BUG_ON(pte == NULL);
  77. ptev = pte_mkwrite(*pte);
  78. if (HYPERVISOR_update_va_mapping(address, ptev, 0))
  79. BUG();
  80. }
  81. void xen_set_pmd(pmd_t *ptr, pmd_t val)
  82. {
  83. struct mmu_update u;
  84. u.ptr = virt_to_machine(ptr).maddr;
  85. u.val = pmd_val_ma(val);
  86. if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0)
  87. BUG();
  88. }
  89. /*
  90. * Associate a virtual page frame with a given physical page frame
  91. * and protection flags for that frame.
  92. */
  93. void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
  94. {
  95. pgd_t *pgd;
  96. pud_t *pud;
  97. pmd_t *pmd;
  98. pte_t *pte;
  99. pgd = swapper_pg_dir + pgd_index(vaddr);
  100. if (pgd_none(*pgd)) {
  101. BUG();
  102. return;
  103. }
  104. pud = pud_offset(pgd, vaddr);
  105. if (pud_none(*pud)) {
  106. BUG();
  107. return;
  108. }
  109. pmd = pmd_offset(pud, vaddr);
  110. if (pmd_none(*pmd)) {
  111. BUG();
  112. return;
  113. }
  114. pte = pte_offset_kernel(pmd, vaddr);
  115. /* <mfn,flags> stored as-is, to permit clearing entries */
  116. xen_set_pte(pte, mfn_pte(mfn, flags));
  117. /*
  118. * It's enough to flush this one mapping.
  119. * (PGE mappings get flushed as well)
  120. */
  121. __flush_tlb_one(vaddr);
  122. }
  123. void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
  124. pte_t *ptep, pte_t pteval)
  125. {
  126. if ((mm != current->mm && mm != &init_mm) ||
  127. HYPERVISOR_update_va_mapping(addr, pteval, 0) != 0)
  128. xen_set_pte(ptep, pteval);
  129. }
  130. #ifdef CONFIG_X86_PAE
  131. void xen_set_pud(pud_t *ptr, pud_t val)
  132. {
  133. struct mmu_update u;
  134. u.ptr = virt_to_machine(ptr).maddr;
  135. u.val = pud_val_ma(val);
  136. if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0)
  137. BUG();
  138. }
  139. void xen_set_pte(pte_t *ptep, pte_t pte)
  140. {
  141. ptep->pte_high = pte.pte_high;
  142. smp_wmb();
  143. ptep->pte_low = pte.pte_low;
  144. }
  145. void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
  146. {
  147. set_64bit((u64 *)ptep, pte_val_ma(pte));
  148. }
  149. void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
  150. {
  151. ptep->pte_low = 0;
  152. smp_wmb(); /* make sure low gets written first */
  153. ptep->pte_high = 0;
  154. }
  155. void xen_pmd_clear(pmd_t *pmdp)
  156. {
  157. xen_set_pmd(pmdp, __pmd(0));
  158. }
  159. unsigned long long xen_pte_val(pte_t pte)
  160. {
  161. unsigned long long ret = 0;
  162. if (pte.pte_low) {
  163. ret = ((unsigned long long)pte.pte_high << 32) | pte.pte_low;
  164. ret = machine_to_phys(XMADDR(ret)).paddr | 1;
  165. }
  166. return ret;
  167. }
  168. unsigned long long xen_pmd_val(pmd_t pmd)
  169. {
  170. unsigned long long ret = pmd.pmd;
  171. if (ret)
  172. ret = machine_to_phys(XMADDR(ret)).paddr | 1;
  173. return ret;
  174. }
  175. unsigned long long xen_pgd_val(pgd_t pgd)
  176. {
  177. unsigned long long ret = pgd.pgd;
  178. if (ret)
  179. ret = machine_to_phys(XMADDR(ret)).paddr | 1;
  180. return ret;
  181. }
  182. pte_t xen_make_pte(unsigned long long pte)
  183. {
  184. if (pte & 1)
  185. pte = phys_to_machine(XPADDR(pte)).maddr;
  186. return (pte_t){ pte, pte >> 32 };
  187. }
  188. pmd_t xen_make_pmd(unsigned long long pmd)
  189. {
  190. if (pmd & 1)
  191. pmd = phys_to_machine(XPADDR(pmd)).maddr;
  192. return (pmd_t){ pmd };
  193. }
  194. pgd_t xen_make_pgd(unsigned long long pgd)
  195. {
  196. if (pgd & _PAGE_PRESENT)
  197. pgd = phys_to_machine(XPADDR(pgd)).maddr;
  198. return (pgd_t){ pgd };
  199. }
  200. #else /* !PAE */
  201. void xen_set_pte(pte_t *ptep, pte_t pte)
  202. {
  203. *ptep = pte;
  204. }
  205. unsigned long xen_pte_val(pte_t pte)
  206. {
  207. unsigned long ret = pte.pte_low;
  208. if (ret & _PAGE_PRESENT)
  209. ret = machine_to_phys(XMADDR(ret)).paddr;
  210. return ret;
  211. }
  212. unsigned long xen_pgd_val(pgd_t pgd)
  213. {
  214. unsigned long ret = pgd.pgd;
  215. if (ret)
  216. ret = machine_to_phys(XMADDR(ret)).paddr | 1;
  217. return ret;
  218. }
  219. pte_t xen_make_pte(unsigned long pte)
  220. {
  221. if (pte & _PAGE_PRESENT)
  222. pte = phys_to_machine(XPADDR(pte)).maddr;
  223. return (pte_t){ pte };
  224. }
  225. pgd_t xen_make_pgd(unsigned long pgd)
  226. {
  227. if (pgd & _PAGE_PRESENT)
  228. pgd = phys_to_machine(XPADDR(pgd)).maddr;
  229. return (pgd_t){ pgd };
  230. }
  231. #endif /* CONFIG_X86_PAE */
  232. /*
  233. (Yet another) pagetable walker. This one is intended for pinning a
  234. pagetable. This means that it walks a pagetable and calls the
  235. callback function on each page it finds making up the page table,
  236. at every level. It walks the entire pagetable, but it only bothers
  237. pinning pte pages which are below pte_limit. In the normal case
  238. this will be TASK_SIZE, but at boot we need to pin up to
  239. FIXADDR_TOP. But the important bit is that we don't pin beyond
  240. there, because then we start getting into Xen's ptes.
  241. */
  242. static int pgd_walk(pgd_t *pgd_base, int (*func)(struct page *, unsigned),
  243. unsigned long limit)
  244. {
  245. pgd_t *pgd = pgd_base;
  246. int flush = 0;
  247. unsigned long addr = 0;
  248. unsigned long pgd_next;
  249. BUG_ON(limit > FIXADDR_TOP);
  250. if (xen_feature(XENFEAT_auto_translated_physmap))
  251. return 0;
  252. for (; addr != FIXADDR_TOP; pgd++, addr = pgd_next) {
  253. pud_t *pud;
  254. unsigned long pud_limit, pud_next;
  255. pgd_next = pud_limit = pgd_addr_end(addr, FIXADDR_TOP);
  256. if (!pgd_val(*pgd))
  257. continue;
  258. pud = pud_offset(pgd, 0);
  259. if (PTRS_PER_PUD > 1) /* not folded */
  260. flush |= (*func)(virt_to_page(pud), 0);
  261. for (; addr != pud_limit; pud++, addr = pud_next) {
  262. pmd_t *pmd;
  263. unsigned long pmd_limit;
  264. pud_next = pud_addr_end(addr, pud_limit);
  265. if (pud_next < limit)
  266. pmd_limit = pud_next;
  267. else
  268. pmd_limit = limit;
  269. if (pud_none(*pud))
  270. continue;
  271. pmd = pmd_offset(pud, 0);
  272. if (PTRS_PER_PMD > 1) /* not folded */
  273. flush |= (*func)(virt_to_page(pmd), 0);
  274. for (; addr != pmd_limit; pmd++) {
  275. addr += (PAGE_SIZE * PTRS_PER_PTE);
  276. if ((pmd_limit-1) < (addr-1)) {
  277. addr = pmd_limit;
  278. break;
  279. }
  280. if (pmd_none(*pmd))
  281. continue;
  282. flush |= (*func)(pmd_page(*pmd), 0);
  283. }
  284. }
  285. }
  286. flush |= (*func)(virt_to_page(pgd_base), UVMF_TLB_FLUSH);
  287. return flush;
  288. }
  289. static int pin_page(struct page *page, unsigned flags)
  290. {
  291. unsigned pgfl = test_and_set_bit(PG_pinned, &page->flags);
  292. int flush;
  293. if (pgfl)
  294. flush = 0; /* already pinned */
  295. else if (PageHighMem(page))
  296. /* kmaps need flushing if we found an unpinned
  297. highpage */
  298. flush = 1;
  299. else {
  300. void *pt = lowmem_page_address(page);
  301. unsigned long pfn = page_to_pfn(page);
  302. struct multicall_space mcs = __xen_mc_entry(0);
  303. flush = 0;
  304. MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
  305. pfn_pte(pfn, PAGE_KERNEL_RO),
  306. flags);
  307. }
  308. return flush;
  309. }
  310. /* This is called just after a mm has been created, but it has not
  311. been used yet. We need to make sure that its pagetable is all
  312. read-only, and can be pinned. */
  313. void xen_pgd_pin(pgd_t *pgd)
  314. {
  315. struct multicall_space mcs;
  316. struct mmuext_op *op;
  317. xen_mc_batch();
  318. if (pgd_walk(pgd, pin_page, TASK_SIZE))
  319. kmap_flush_unused();
  320. mcs = __xen_mc_entry(sizeof(*op));
  321. op = mcs.args;
  322. #ifdef CONFIG_X86_PAE
  323. op->cmd = MMUEXT_PIN_L3_TABLE;
  324. #else
  325. op->cmd = MMUEXT_PIN_L2_TABLE;
  326. #endif
  327. op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
  328. MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
  329. xen_mc_issue(0);
  330. }
  331. /* The init_mm pagetable is really pinned as soon as its created, but
  332. that's before we have page structures to store the bits. So do all
  333. the book-keeping now. */
  334. static __init int mark_pinned(struct page *page, unsigned flags)
  335. {
  336. SetPagePinned(page);
  337. return 0;
  338. }
  339. void __init xen_mark_init_mm_pinned(void)
  340. {
  341. pgd_walk(init_mm.pgd, mark_pinned, FIXADDR_TOP);
  342. }
  343. static int unpin_page(struct page *page, unsigned flags)
  344. {
  345. unsigned pgfl = test_and_clear_bit(PG_pinned, &page->flags);
  346. if (pgfl && !PageHighMem(page)) {
  347. void *pt = lowmem_page_address(page);
  348. unsigned long pfn = page_to_pfn(page);
  349. struct multicall_space mcs = __xen_mc_entry(0);
  350. MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
  351. pfn_pte(pfn, PAGE_KERNEL),
  352. flags);
  353. }
  354. return 0; /* never need to flush on unpin */
  355. }
  356. /* Release a pagetables pages back as normal RW */
  357. static void xen_pgd_unpin(pgd_t *pgd)
  358. {
  359. struct mmuext_op *op;
  360. struct multicall_space mcs;
  361. xen_mc_batch();
  362. mcs = __xen_mc_entry(sizeof(*op));
  363. op = mcs.args;
  364. op->cmd = MMUEXT_UNPIN_TABLE;
  365. op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
  366. MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
  367. pgd_walk(pgd, unpin_page, TASK_SIZE);
  368. xen_mc_issue(0);
  369. }
  370. void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
  371. {
  372. spin_lock(&next->page_table_lock);
  373. xen_pgd_pin(next->pgd);
  374. spin_unlock(&next->page_table_lock);
  375. }
  376. void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
  377. {
  378. spin_lock(&mm->page_table_lock);
  379. xen_pgd_pin(mm->pgd);
  380. spin_unlock(&mm->page_table_lock);
  381. }
  382. void xen_exit_mmap(struct mm_struct *mm)
  383. {
  384. struct task_struct *tsk = current;
  385. task_lock(tsk);
  386. /*
  387. * We aggressively remove defunct pgd from cr3. We execute unmap_vmas()
  388. * *much* faster this way, as no tlb flushes means bigger wrpt batches.
  389. */
  390. if (tsk->active_mm == mm) {
  391. tsk->active_mm = &init_mm;
  392. atomic_inc(&init_mm.mm_count);
  393. switch_mm(mm, &init_mm, tsk);
  394. atomic_dec(&mm->mm_count);
  395. BUG_ON(atomic_read(&mm->mm_count) == 0);
  396. }
  397. task_unlock(tsk);
  398. xen_pgd_unpin(mm->pgd);
  399. }