mmu.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841
  1. /*
  2. * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  3. * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License, version 2, as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  17. */
  18. #include <linux/mman.h>
  19. #include <linux/kvm_host.h>
  20. #include <linux/io.h>
  21. #include <trace/events/kvm.h>
  22. #include <asm/pgalloc.h>
  23. #include <asm/cacheflush.h>
  24. #include <asm/kvm_arm.h>
  25. #include <asm/kvm_mmu.h>
  26. #include <asm/kvm_mmio.h>
  27. #include <asm/kvm_asm.h>
  28. #include <asm/kvm_emulate.h>
  29. #include "trace.h"
  30. extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
  31. static pgd_t *boot_hyp_pgd;
  32. static pgd_t *hyp_pgd;
  33. static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
  34. static void *init_bounce_page;
  35. static unsigned long hyp_idmap_start;
  36. static unsigned long hyp_idmap_end;
  37. static phys_addr_t hyp_idmap_vector;
  38. static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
  39. {
  40. /*
  41. * This function also gets called when dealing with HYP page
  42. * tables. As HYP doesn't have an associated struct kvm (and
  43. * the HYP page tables are fairly static), we don't do
  44. * anything there.
  45. */
  46. if (kvm)
  47. kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
  48. }
  49. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  50. int min, int max)
  51. {
  52. void *page;
  53. BUG_ON(max > KVM_NR_MEM_OBJS);
  54. if (cache->nobjs >= min)
  55. return 0;
  56. while (cache->nobjs < max) {
  57. page = (void *)__get_free_page(PGALLOC_GFP);
  58. if (!page)
  59. return -ENOMEM;
  60. cache->objects[cache->nobjs++] = page;
  61. }
  62. return 0;
  63. }
  64. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
  65. {
  66. while (mc->nobjs)
  67. free_page((unsigned long)mc->objects[--mc->nobjs]);
  68. }
  69. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
  70. {
  71. void *p;
  72. BUG_ON(!mc || !mc->nobjs);
  73. p = mc->objects[--mc->nobjs];
  74. return p;
  75. }
  76. static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
  77. {
  78. pmd_t *pmd_table = pmd_offset(pud, 0);
  79. pud_clear(pud);
  80. kvm_tlb_flush_vmid_ipa(kvm, addr);
  81. pmd_free(NULL, pmd_table);
  82. put_page(virt_to_page(pud));
  83. }
  84. static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
  85. {
  86. pte_t *pte_table = pte_offset_kernel(pmd, 0);
  87. pmd_clear(pmd);
  88. kvm_tlb_flush_vmid_ipa(kvm, addr);
  89. pte_free_kernel(NULL, pte_table);
  90. put_page(virt_to_page(pmd));
  91. }
  92. static bool pmd_empty(pmd_t *pmd)
  93. {
  94. struct page *pmd_page = virt_to_page(pmd);
  95. return page_count(pmd_page) == 1;
  96. }
  97. static void clear_pte_entry(struct kvm *kvm, pte_t *pte, phys_addr_t addr)
  98. {
  99. if (pte_present(*pte)) {
  100. kvm_set_pte(pte, __pte(0));
  101. put_page(virt_to_page(pte));
  102. kvm_tlb_flush_vmid_ipa(kvm, addr);
  103. }
  104. }
  105. static bool pte_empty(pte_t *pte)
  106. {
  107. struct page *pte_page = virt_to_page(pte);
  108. return page_count(pte_page) == 1;
  109. }
  110. static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
  111. unsigned long long start, u64 size)
  112. {
  113. pgd_t *pgd;
  114. pud_t *pud;
  115. pmd_t *pmd;
  116. pte_t *pte;
  117. unsigned long long addr = start, end = start + size;
  118. u64 range;
  119. while (addr < end) {
  120. pgd = pgdp + pgd_index(addr);
  121. pud = pud_offset(pgd, addr);
  122. if (pud_none(*pud)) {
  123. addr += PUD_SIZE;
  124. continue;
  125. }
  126. pmd = pmd_offset(pud, addr);
  127. if (pmd_none(*pmd)) {
  128. addr += PMD_SIZE;
  129. continue;
  130. }
  131. pte = pte_offset_kernel(pmd, addr);
  132. clear_pte_entry(kvm, pte, addr);
  133. range = PAGE_SIZE;
  134. /* If we emptied the pte, walk back up the ladder */
  135. if (pte_empty(pte)) {
  136. clear_pmd_entry(kvm, pmd, addr);
  137. range = PMD_SIZE;
  138. if (pmd_empty(pmd)) {
  139. clear_pud_entry(kvm, pud, addr);
  140. range = PUD_SIZE;
  141. }
  142. }
  143. addr += range;
  144. }
  145. }
  146. /**
  147. * free_boot_hyp_pgd - free HYP boot page tables
  148. *
  149. * Free the HYP boot page tables. The bounce page is also freed.
  150. */
  151. void free_boot_hyp_pgd(void)
  152. {
  153. mutex_lock(&kvm_hyp_pgd_mutex);
  154. if (boot_hyp_pgd) {
  155. unmap_range(NULL, boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
  156. unmap_range(NULL, boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
  157. kfree(boot_hyp_pgd);
  158. boot_hyp_pgd = NULL;
  159. }
  160. if (hyp_pgd)
  161. unmap_range(NULL, hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
  162. kfree(init_bounce_page);
  163. init_bounce_page = NULL;
  164. mutex_unlock(&kvm_hyp_pgd_mutex);
  165. }
  166. /**
  167. * free_hyp_pgds - free Hyp-mode page tables
  168. *
  169. * Assumes hyp_pgd is a page table used strictly in Hyp-mode and
  170. * therefore contains either mappings in the kernel memory area (above
  171. * PAGE_OFFSET), or device mappings in the vmalloc range (from
  172. * VMALLOC_START to VMALLOC_END).
  173. *
  174. * boot_hyp_pgd should only map two pages for the init code.
  175. */
  176. void free_hyp_pgds(void)
  177. {
  178. unsigned long addr;
  179. free_boot_hyp_pgd();
  180. mutex_lock(&kvm_hyp_pgd_mutex);
  181. if (hyp_pgd) {
  182. for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)
  183. unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
  184. for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)
  185. unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
  186. kfree(hyp_pgd);
  187. hyp_pgd = NULL;
  188. }
  189. mutex_unlock(&kvm_hyp_pgd_mutex);
  190. }
  191. static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
  192. unsigned long end, unsigned long pfn,
  193. pgprot_t prot)
  194. {
  195. pte_t *pte;
  196. unsigned long addr;
  197. addr = start;
  198. do {
  199. pte = pte_offset_kernel(pmd, addr);
  200. kvm_set_pte(pte, pfn_pte(pfn, prot));
  201. get_page(virt_to_page(pte));
  202. kvm_flush_dcache_to_poc(pte, sizeof(*pte));
  203. pfn++;
  204. } while (addr += PAGE_SIZE, addr != end);
  205. }
  206. static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
  207. unsigned long end, unsigned long pfn,
  208. pgprot_t prot)
  209. {
  210. pmd_t *pmd;
  211. pte_t *pte;
  212. unsigned long addr, next;
  213. addr = start;
  214. do {
  215. pmd = pmd_offset(pud, addr);
  216. BUG_ON(pmd_sect(*pmd));
  217. if (pmd_none(*pmd)) {
  218. pte = pte_alloc_one_kernel(NULL, addr);
  219. if (!pte) {
  220. kvm_err("Cannot allocate Hyp pte\n");
  221. return -ENOMEM;
  222. }
  223. pmd_populate_kernel(NULL, pmd, pte);
  224. get_page(virt_to_page(pmd));
  225. kvm_flush_dcache_to_poc(pmd, sizeof(*pmd));
  226. }
  227. next = pmd_addr_end(addr, end);
  228. create_hyp_pte_mappings(pmd, addr, next, pfn, prot);
  229. pfn += (next - addr) >> PAGE_SHIFT;
  230. } while (addr = next, addr != end);
  231. return 0;
  232. }
  233. static int __create_hyp_mappings(pgd_t *pgdp,
  234. unsigned long start, unsigned long end,
  235. unsigned long pfn, pgprot_t prot)
  236. {
  237. pgd_t *pgd;
  238. pud_t *pud;
  239. pmd_t *pmd;
  240. unsigned long addr, next;
  241. int err = 0;
  242. mutex_lock(&kvm_hyp_pgd_mutex);
  243. addr = start & PAGE_MASK;
  244. end = PAGE_ALIGN(end);
  245. do {
  246. pgd = pgdp + pgd_index(addr);
  247. pud = pud_offset(pgd, addr);
  248. if (pud_none_or_clear_bad(pud)) {
  249. pmd = pmd_alloc_one(NULL, addr);
  250. if (!pmd) {
  251. kvm_err("Cannot allocate Hyp pmd\n");
  252. err = -ENOMEM;
  253. goto out;
  254. }
  255. pud_populate(NULL, pud, pmd);
  256. get_page(virt_to_page(pud));
  257. kvm_flush_dcache_to_poc(pud, sizeof(*pud));
  258. }
  259. next = pgd_addr_end(addr, end);
  260. err = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);
  261. if (err)
  262. goto out;
  263. pfn += (next - addr) >> PAGE_SHIFT;
  264. } while (addr = next, addr != end);
  265. out:
  266. mutex_unlock(&kvm_hyp_pgd_mutex);
  267. return err;
  268. }
  269. /**
  270. * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
  271. * @from: The virtual kernel start address of the range
  272. * @to: The virtual kernel end address of the range (exclusive)
  273. *
  274. * The same virtual address as the kernel virtual address is also used
  275. * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
  276. * physical pages.
  277. */
  278. int create_hyp_mappings(void *from, void *to)
  279. {
  280. unsigned long phys_addr = virt_to_phys(from);
  281. unsigned long start = KERN_TO_HYP((unsigned long)from);
  282. unsigned long end = KERN_TO_HYP((unsigned long)to);
  283. /* Check for a valid kernel memory mapping */
  284. if (!virt_addr_valid(from) || !virt_addr_valid(to - 1))
  285. return -EINVAL;
  286. return __create_hyp_mappings(hyp_pgd, start, end,
  287. __phys_to_pfn(phys_addr), PAGE_HYP);
  288. }
  289. /**
  290. * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode
  291. * @from: The kernel start VA of the range
  292. * @to: The kernel end VA of the range (exclusive)
  293. * @phys_addr: The physical start address which gets mapped
  294. *
  295. * The resulting HYP VA is the same as the kernel VA, modulo
  296. * HYP_PAGE_OFFSET.
  297. */
  298. int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
  299. {
  300. unsigned long start = KERN_TO_HYP((unsigned long)from);
  301. unsigned long end = KERN_TO_HYP((unsigned long)to);
  302. /* Check for a valid kernel IO mapping */
  303. if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1))
  304. return -EINVAL;
  305. return __create_hyp_mappings(hyp_pgd, start, end,
  306. __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
  307. }
  308. /**
  309. * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
  310. * @kvm: The KVM struct pointer for the VM.
  311. *
  312. * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can
  313. * support either full 40-bit input addresses or limited to 32-bit input
  314. * addresses). Clears the allocated pages.
  315. *
  316. * Note we don't need locking here as this is only called when the VM is
  317. * created, which can only be done once.
  318. */
  319. int kvm_alloc_stage2_pgd(struct kvm *kvm)
  320. {
  321. pgd_t *pgd;
  322. if (kvm->arch.pgd != NULL) {
  323. kvm_err("kvm_arch already initialized?\n");
  324. return -EINVAL;
  325. }
  326. pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, S2_PGD_ORDER);
  327. if (!pgd)
  328. return -ENOMEM;
  329. memset(pgd, 0, PTRS_PER_S2_PGD * sizeof(pgd_t));
  330. kvm_clean_pgd(pgd);
  331. kvm->arch.pgd = pgd;
  332. return 0;
  333. }
  334. /**
  335. * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
  336. * @kvm: The VM pointer
  337. * @start: The intermediate physical base address of the range to unmap
  338. * @size: The size of the area to unmap
  339. *
  340. * Clear a range of stage-2 mappings, lowering the various ref-counts. Must
  341. * be called while holding mmu_lock (unless for freeing the stage2 pgd before
  342. * destroying the VM), otherwise another faulting VCPU may come in and mess
  343. * with things behind our backs.
  344. */
  345. static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
  346. {
  347. unmap_range(kvm, kvm->arch.pgd, start, size);
  348. }
  349. /**
  350. * kvm_free_stage2_pgd - free all stage-2 tables
  351. * @kvm: The KVM struct pointer for the VM.
  352. *
  353. * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
  354. * underlying level-2 and level-3 tables before freeing the actual level-1 table
  355. * and setting the struct pointer to NULL.
  356. *
  357. * Note we don't need locking here as this is only called when the VM is
  358. * destroyed, which can only be done once.
  359. */
  360. void kvm_free_stage2_pgd(struct kvm *kvm)
  361. {
  362. if (kvm->arch.pgd == NULL)
  363. return;
  364. unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
  365. free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER);
  366. kvm->arch.pgd = NULL;
  367. }
  368. static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
  369. phys_addr_t addr, const pte_t *new_pte, bool iomap)
  370. {
  371. pgd_t *pgd;
  372. pud_t *pud;
  373. pmd_t *pmd;
  374. pte_t *pte, old_pte;
  375. /* Create 2nd stage page table mapping - Level 1 */
  376. pgd = kvm->arch.pgd + pgd_index(addr);
  377. pud = pud_offset(pgd, addr);
  378. if (pud_none(*pud)) {
  379. if (!cache)
  380. return 0; /* ignore calls from kvm_set_spte_hva */
  381. pmd = mmu_memory_cache_alloc(cache);
  382. pud_populate(NULL, pud, pmd);
  383. get_page(virt_to_page(pud));
  384. }
  385. pmd = pmd_offset(pud, addr);
  386. /* Create 2nd stage page table mapping - Level 2 */
  387. if (pmd_none(*pmd)) {
  388. if (!cache)
  389. return 0; /* ignore calls from kvm_set_spte_hva */
  390. pte = mmu_memory_cache_alloc(cache);
  391. kvm_clean_pte(pte);
  392. pmd_populate_kernel(NULL, pmd, pte);
  393. get_page(virt_to_page(pmd));
  394. }
  395. pte = pte_offset_kernel(pmd, addr);
  396. if (iomap && pte_present(*pte))
  397. return -EFAULT;
  398. /* Create 2nd stage page table mapping - Level 3 */
  399. old_pte = *pte;
  400. kvm_set_pte(pte, *new_pte);
  401. if (pte_present(old_pte))
  402. kvm_tlb_flush_vmid_ipa(kvm, addr);
  403. else
  404. get_page(virt_to_page(pte));
  405. return 0;
  406. }
  407. /**
  408. * kvm_phys_addr_ioremap - map a device range to guest IPA
  409. *
  410. * @kvm: The KVM pointer
  411. * @guest_ipa: The IPA at which to insert the mapping
  412. * @pa: The physical address of the device
  413. * @size: The size of the mapping
  414. */
  415. int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
  416. phys_addr_t pa, unsigned long size)
  417. {
  418. phys_addr_t addr, end;
  419. int ret = 0;
  420. unsigned long pfn;
  421. struct kvm_mmu_memory_cache cache = { 0, };
  422. end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
  423. pfn = __phys_to_pfn(pa);
  424. for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
  425. pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE);
  426. kvm_set_s2pte_writable(&pte);
  427. ret = mmu_topup_memory_cache(&cache, 2, 2);
  428. if (ret)
  429. goto out;
  430. spin_lock(&kvm->mmu_lock);
  431. ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
  432. spin_unlock(&kvm->mmu_lock);
  433. if (ret)
  434. goto out;
  435. pfn++;
  436. }
  437. out:
  438. mmu_free_memory_cache(&cache);
  439. return ret;
  440. }
  441. static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
  442. gfn_t gfn, struct kvm_memory_slot *memslot,
  443. unsigned long fault_status)
  444. {
  445. pte_t new_pte;
  446. pfn_t pfn;
  447. int ret;
  448. bool write_fault, writable;
  449. unsigned long mmu_seq;
  450. struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
  451. write_fault = kvm_is_write_fault(kvm_vcpu_get_hsr(vcpu));
  452. if (fault_status == FSC_PERM && !write_fault) {
  453. kvm_err("Unexpected L2 read permission error\n");
  454. return -EFAULT;
  455. }
  456. /* We need minimum second+third level pages */
  457. ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS);
  458. if (ret)
  459. return ret;
  460. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  461. /*
  462. * Ensure the read of mmu_notifier_seq happens before we call
  463. * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
  464. * the page we just got a reference to gets unmapped before we have a
  465. * chance to grab the mmu_lock, which ensure that if the page gets
  466. * unmapped afterwards, the call to kvm_unmap_hva will take it away
  467. * from us again properly. This smp_rmb() interacts with the smp_wmb()
  468. * in kvm_mmu_notifier_invalidate_<page|range_end>.
  469. */
  470. smp_rmb();
  471. pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write_fault, &writable);
  472. if (is_error_pfn(pfn))
  473. return -EFAULT;
  474. new_pte = pfn_pte(pfn, PAGE_S2);
  475. coherent_icache_guest_page(vcpu->kvm, gfn);
  476. spin_lock(&vcpu->kvm->mmu_lock);
  477. if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
  478. goto out_unlock;
  479. if (writable) {
  480. kvm_set_s2pte_writable(&new_pte);
  481. kvm_set_pfn_dirty(pfn);
  482. }
  483. stage2_set_pte(vcpu->kvm, memcache, fault_ipa, &new_pte, false);
  484. out_unlock:
  485. spin_unlock(&vcpu->kvm->mmu_lock);
  486. kvm_release_pfn_clean(pfn);
  487. return 0;
  488. }
  489. /**
  490. * kvm_handle_guest_abort - handles all 2nd stage aborts
  491. * @vcpu: the VCPU pointer
  492. * @run: the kvm_run structure
  493. *
  494. * Any abort that gets to the host is almost guaranteed to be caused by a
  495. * missing second stage translation table entry, which can mean that either the
  496. * guest simply needs more memory and we must allocate an appropriate page or it
  497. * can mean that the guest tried to access I/O memory, which is emulated by user
  498. * space. The distinction is based on the IPA causing the fault and whether this
  499. * memory region has been registered as standard RAM by user space.
  500. */
  501. int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
  502. {
  503. unsigned long fault_status;
  504. phys_addr_t fault_ipa;
  505. struct kvm_memory_slot *memslot;
  506. bool is_iabt;
  507. gfn_t gfn;
  508. int ret, idx;
  509. is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
  510. fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
  511. trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
  512. kvm_vcpu_get_hfar(vcpu), fault_ipa);
  513. /* Check the stage-2 fault is trans. fault or write fault */
  514. fault_status = kvm_vcpu_trap_get_fault(vcpu);
  515. if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
  516. kvm_err("Unsupported fault status: EC=%#x DFCS=%#lx\n",
  517. kvm_vcpu_trap_get_class(vcpu), fault_status);
  518. return -EFAULT;
  519. }
  520. idx = srcu_read_lock(&vcpu->kvm->srcu);
  521. gfn = fault_ipa >> PAGE_SHIFT;
  522. if (!kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  523. if (is_iabt) {
  524. /* Prefetch Abort on I/O address */
  525. kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
  526. ret = 1;
  527. goto out_unlock;
  528. }
  529. if (fault_status != FSC_FAULT) {
  530. kvm_err("Unsupported fault status on io memory: %#lx\n",
  531. fault_status);
  532. ret = -EFAULT;
  533. goto out_unlock;
  534. }
  535. /*
  536. * The IPA is reported as [MAX:12], so we need to
  537. * complement it with the bottom 12 bits from the
  538. * faulting VA. This is always 12 bits, irrespective
  539. * of the page size.
  540. */
  541. fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
  542. ret = io_mem_abort(vcpu, run, fault_ipa);
  543. goto out_unlock;
  544. }
  545. memslot = gfn_to_memslot(vcpu->kvm, gfn);
  546. ret = user_mem_abort(vcpu, fault_ipa, gfn, memslot, fault_status);
  547. if (ret == 0)
  548. ret = 1;
  549. out_unlock:
  550. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  551. return ret;
  552. }
  553. static void handle_hva_to_gpa(struct kvm *kvm,
  554. unsigned long start,
  555. unsigned long end,
  556. void (*handler)(struct kvm *kvm,
  557. gpa_t gpa, void *data),
  558. void *data)
  559. {
  560. struct kvm_memslots *slots;
  561. struct kvm_memory_slot *memslot;
  562. slots = kvm_memslots(kvm);
  563. /* we only care about the pages that the guest sees */
  564. kvm_for_each_memslot(memslot, slots) {
  565. unsigned long hva_start, hva_end;
  566. gfn_t gfn, gfn_end;
  567. hva_start = max(start, memslot->userspace_addr);
  568. hva_end = min(end, memslot->userspace_addr +
  569. (memslot->npages << PAGE_SHIFT));
  570. if (hva_start >= hva_end)
  571. continue;
  572. /*
  573. * {gfn(page) | page intersects with [hva_start, hva_end)} =
  574. * {gfn_start, gfn_start+1, ..., gfn_end-1}.
  575. */
  576. gfn = hva_to_gfn_memslot(hva_start, memslot);
  577. gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
  578. for (; gfn < gfn_end; ++gfn) {
  579. gpa_t gpa = gfn << PAGE_SHIFT;
  580. handler(kvm, gpa, data);
  581. }
  582. }
  583. }
  584. static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
  585. {
  586. unmap_stage2_range(kvm, gpa, PAGE_SIZE);
  587. }
  588. int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
  589. {
  590. unsigned long end = hva + PAGE_SIZE;
  591. if (!kvm->arch.pgd)
  592. return 0;
  593. trace_kvm_unmap_hva(hva);
  594. handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
  595. return 0;
  596. }
  597. int kvm_unmap_hva_range(struct kvm *kvm,
  598. unsigned long start, unsigned long end)
  599. {
  600. if (!kvm->arch.pgd)
  601. return 0;
  602. trace_kvm_unmap_hva_range(start, end);
  603. handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
  604. return 0;
  605. }
  606. static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
  607. {
  608. pte_t *pte = (pte_t *)data;
  609. stage2_set_pte(kvm, NULL, gpa, pte, false);
  610. }
  611. void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
  612. {
  613. unsigned long end = hva + PAGE_SIZE;
  614. pte_t stage2_pte;
  615. if (!kvm->arch.pgd)
  616. return;
  617. trace_kvm_set_spte_hva(hva);
  618. stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
  619. handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
  620. }
  621. void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  622. {
  623. mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
  624. }
  625. phys_addr_t kvm_mmu_get_httbr(void)
  626. {
  627. return virt_to_phys(hyp_pgd);
  628. }
  629. phys_addr_t kvm_mmu_get_boot_httbr(void)
  630. {
  631. return virt_to_phys(boot_hyp_pgd);
  632. }
  633. phys_addr_t kvm_get_idmap_vector(void)
  634. {
  635. return hyp_idmap_vector;
  636. }
  637. int kvm_mmu_init(void)
  638. {
  639. int err;
  640. hyp_idmap_start = virt_to_phys(__hyp_idmap_text_start);
  641. hyp_idmap_end = virt_to_phys(__hyp_idmap_text_end);
  642. hyp_idmap_vector = virt_to_phys(__kvm_hyp_init);
  643. if ((hyp_idmap_start ^ hyp_idmap_end) & PAGE_MASK) {
  644. /*
  645. * Our init code is crossing a page boundary. Allocate
  646. * a bounce page, copy the code over and use that.
  647. */
  648. size_t len = __hyp_idmap_text_end - __hyp_idmap_text_start;
  649. phys_addr_t phys_base;
  650. init_bounce_page = kmalloc(PAGE_SIZE, GFP_KERNEL);
  651. if (!init_bounce_page) {
  652. kvm_err("Couldn't allocate HYP init bounce page\n");
  653. err = -ENOMEM;
  654. goto out;
  655. }
  656. memcpy(init_bounce_page, __hyp_idmap_text_start, len);
  657. /*
  658. * Warning: the code we just copied to the bounce page
  659. * must be flushed to the point of coherency.
  660. * Otherwise, the data may be sitting in L2, and HYP
  661. * mode won't be able to observe it as it runs with
  662. * caches off at that point.
  663. */
  664. kvm_flush_dcache_to_poc(init_bounce_page, len);
  665. phys_base = virt_to_phys(init_bounce_page);
  666. hyp_idmap_vector += phys_base - hyp_idmap_start;
  667. hyp_idmap_start = phys_base;
  668. hyp_idmap_end = phys_base + len;
  669. kvm_info("Using HYP init bounce page @%lx\n",
  670. (unsigned long)phys_base);
  671. }
  672. hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
  673. boot_hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
  674. if (!hyp_pgd || !boot_hyp_pgd) {
  675. kvm_err("Hyp mode PGD not allocated\n");
  676. err = -ENOMEM;
  677. goto out;
  678. }
  679. /* Create the idmap in the boot page tables */
  680. err = __create_hyp_mappings(boot_hyp_pgd,
  681. hyp_idmap_start, hyp_idmap_end,
  682. __phys_to_pfn(hyp_idmap_start),
  683. PAGE_HYP);
  684. if (err) {
  685. kvm_err("Failed to idmap %lx-%lx\n",
  686. hyp_idmap_start, hyp_idmap_end);
  687. goto out;
  688. }
  689. /* Map the very same page at the trampoline VA */
  690. err = __create_hyp_mappings(boot_hyp_pgd,
  691. TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
  692. __phys_to_pfn(hyp_idmap_start),
  693. PAGE_HYP);
  694. if (err) {
  695. kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n",
  696. TRAMPOLINE_VA);
  697. goto out;
  698. }
  699. /* Map the same page again into the runtime page tables */
  700. err = __create_hyp_mappings(hyp_pgd,
  701. TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
  702. __phys_to_pfn(hyp_idmap_start),
  703. PAGE_HYP);
  704. if (err) {
  705. kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n",
  706. TRAMPOLINE_VA);
  707. goto out;
  708. }
  709. return 0;
  710. out:
  711. free_hyp_pgds();
  712. return err;
  713. }