e500_tlb.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377
  1. /*
  2. * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
  3. *
  4. * Author: Yu Liu, yu.liu@freescale.com
  5. * Scott Wood, scottwood@freescale.com
  6. * Ashish Kalra, ashish.kalra@freescale.com
  7. * Varun Sethi, varun.sethi@freescale.com
  8. *
  9. * Description:
  10. * This file is based on arch/powerpc/kvm/44x_tlb.c,
  11. * by Hollis Blanchard <hollisb@us.ibm.com>.
  12. *
  13. * This program is free software; you can redistribute it and/or modify
  14. * it under the terms of the GNU General Public License, version 2, as
  15. * published by the Free Software Foundation.
  16. */
  17. #include <linux/kernel.h>
  18. #include <linux/types.h>
  19. #include <linux/slab.h>
  20. #include <linux/string.h>
  21. #include <linux/kvm.h>
  22. #include <linux/kvm_host.h>
  23. #include <linux/highmem.h>
  24. #include <linux/log2.h>
  25. #include <linux/uaccess.h>
  26. #include <linux/sched.h>
  27. #include <linux/rwsem.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/hugetlb.h>
  30. #include <asm/kvm_ppc.h>
  31. #include "e500.h"
  32. #include "trace.h"
  33. #include "timing.h"
  34. #define to_htlb1_esel(esel) (host_tlb_params[1].entries - (esel) - 1)
  35. static struct kvmppc_e500_tlb_params host_tlb_params[E500_TLB_NUM];
  36. static inline unsigned int gtlb0_get_next_victim(
  37. struct kvmppc_vcpu_e500 *vcpu_e500)
  38. {
  39. unsigned int victim;
  40. victim = vcpu_e500->gtlb_nv[0]++;
  41. if (unlikely(vcpu_e500->gtlb_nv[0] >= vcpu_e500->gtlb_params[0].ways))
  42. vcpu_e500->gtlb_nv[0] = 0;
  43. return victim;
  44. }
  45. static inline unsigned int tlb1_max_shadow_size(void)
  46. {
  47. /* reserve one entry for magic page */
  48. return host_tlb_params[1].entries - tlbcam_index - 1;
  49. }
  50. static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry *tlbe)
  51. {
  52. return tlbe->mas7_3 & (MAS3_SW|MAS3_UW);
  53. }
  54. static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode)
  55. {
  56. /* Mask off reserved bits. */
  57. mas3 &= MAS3_ATTRIB_MASK;
  58. #ifndef CONFIG_KVM_BOOKE_HV
  59. if (!usermode) {
  60. /* Guest is in supervisor mode,
  61. * so we need to translate guest
  62. * supervisor permissions into user permissions. */
  63. mas3 &= ~E500_TLB_USER_PERM_MASK;
  64. mas3 |= (mas3 & E500_TLB_SUPER_PERM_MASK) << 1;
  65. }
  66. mas3 |= E500_TLB_SUPER_PERM_MASK;
  67. #endif
  68. return mas3;
  69. }
  70. static inline u32 e500_shadow_mas2_attrib(u32 mas2, int usermode)
  71. {
  72. #ifdef CONFIG_SMP
  73. return (mas2 & MAS2_ATTRIB_MASK) | MAS2_M;
  74. #else
  75. return mas2 & MAS2_ATTRIB_MASK;
  76. #endif
  77. }
  78. /*
  79. * writing shadow tlb entry to host TLB
  80. */
  81. static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry *stlbe,
  82. uint32_t mas0)
  83. {
  84. unsigned long flags;
  85. local_irq_save(flags);
  86. mtspr(SPRN_MAS0, mas0);
  87. mtspr(SPRN_MAS1, stlbe->mas1);
  88. mtspr(SPRN_MAS2, (unsigned long)stlbe->mas2);
  89. mtspr(SPRN_MAS3, (u32)stlbe->mas7_3);
  90. mtspr(SPRN_MAS7, (u32)(stlbe->mas7_3 >> 32));
  91. #ifdef CONFIG_KVM_BOOKE_HV
  92. mtspr(SPRN_MAS8, stlbe->mas8);
  93. #endif
  94. asm volatile("isync; tlbwe" : : : "memory");
  95. #ifdef CONFIG_KVM_BOOKE_HV
  96. /* Must clear mas8 for other host tlbwe's */
  97. mtspr(SPRN_MAS8, 0);
  98. isync();
  99. #endif
  100. local_irq_restore(flags);
  101. trace_kvm_booke206_stlb_write(mas0, stlbe->mas8, stlbe->mas1,
  102. stlbe->mas2, stlbe->mas7_3);
  103. }
  104. /*
  105. * Acquire a mas0 with victim hint, as if we just took a TLB miss.
  106. *
  107. * We don't care about the address we're searching for, other than that it's
  108. * in the right set and is not present in the TLB. Using a zero PID and a
  109. * userspace address means we don't have to set and then restore MAS5, or
  110. * calculate a proper MAS6 value.
  111. */
  112. static u32 get_host_mas0(unsigned long eaddr)
  113. {
  114. unsigned long flags;
  115. u32 mas0;
  116. local_irq_save(flags);
  117. mtspr(SPRN_MAS6, 0);
  118. asm volatile("tlbsx 0, %0" : : "b" (eaddr & ~CONFIG_PAGE_OFFSET));
  119. mas0 = mfspr(SPRN_MAS0);
  120. local_irq_restore(flags);
  121. return mas0;
  122. }
  123. /* sesel is for tlb1 only */
  124. static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
  125. int tlbsel, int sesel, struct kvm_book3e_206_tlb_entry *stlbe)
  126. {
  127. u32 mas0;
  128. if (tlbsel == 0) {
  129. mas0 = get_host_mas0(stlbe->mas2);
  130. __write_host_tlbe(stlbe, mas0);
  131. } else {
  132. __write_host_tlbe(stlbe,
  133. MAS0_TLBSEL(1) |
  134. MAS0_ESEL(to_htlb1_esel(sesel)));
  135. }
  136. }
  137. #ifdef CONFIG_KVM_E500V2
  138. void kvmppc_map_magic(struct kvm_vcpu *vcpu)
  139. {
  140. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  141. struct kvm_book3e_206_tlb_entry magic;
  142. ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK;
  143. unsigned int stid;
  144. pfn_t pfn;
  145. pfn = (pfn_t)virt_to_phys((void *)shared_page) >> PAGE_SHIFT;
  146. get_page(pfn_to_page(pfn));
  147. preempt_disable();
  148. stid = kvmppc_e500_get_sid(vcpu_e500, 0, 0, 0, 0);
  149. magic.mas1 = MAS1_VALID | MAS1_TS | MAS1_TID(stid) |
  150. MAS1_TSIZE(BOOK3E_PAGESZ_4K);
  151. magic.mas2 = vcpu->arch.magic_page_ea | MAS2_M;
  152. magic.mas7_3 = ((u64)pfn << PAGE_SHIFT) |
  153. MAS3_SW | MAS3_SR | MAS3_UW | MAS3_UR;
  154. magic.mas8 = 0;
  155. __write_host_tlbe(&magic, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index));
  156. preempt_enable();
  157. }
  158. #endif
  159. static void inval_gtlbe_on_host(struct kvmppc_vcpu_e500 *vcpu_e500,
  160. int tlbsel, int esel)
  161. {
  162. struct kvm_book3e_206_tlb_entry *gtlbe =
  163. get_entry(vcpu_e500, tlbsel, esel);
  164. if (tlbsel == 1 &&
  165. vcpu_e500->gtlb_priv[1][esel].ref.flags & E500_TLB_BITMAP) {
  166. u64 tmp = vcpu_e500->g2h_tlb1_map[esel];
  167. int hw_tlb_indx;
  168. unsigned long flags;
  169. local_irq_save(flags);
  170. while (tmp) {
  171. hw_tlb_indx = __ilog2_u64(tmp & -tmp);
  172. mtspr(SPRN_MAS0,
  173. MAS0_TLBSEL(1) |
  174. MAS0_ESEL(to_htlb1_esel(hw_tlb_indx)));
  175. mtspr(SPRN_MAS1, 0);
  176. asm volatile("tlbwe");
  177. vcpu_e500->h2g_tlb1_rmap[hw_tlb_indx] = 0;
  178. tmp &= tmp - 1;
  179. }
  180. mb();
  181. vcpu_e500->g2h_tlb1_map[esel] = 0;
  182. vcpu_e500->gtlb_priv[1][esel].ref.flags &= ~E500_TLB_BITMAP;
  183. local_irq_restore(flags);
  184. return;
  185. }
  186. /* Guest tlbe is backed by at most one host tlbe per shadow pid. */
  187. kvmppc_e500_tlbil_one(vcpu_e500, gtlbe);
  188. }
  189. static int tlb0_set_base(gva_t addr, int sets, int ways)
  190. {
  191. int set_base;
  192. set_base = (addr >> PAGE_SHIFT) & (sets - 1);
  193. set_base *= ways;
  194. return set_base;
  195. }
  196. static int gtlb0_set_base(struct kvmppc_vcpu_e500 *vcpu_e500, gva_t addr)
  197. {
  198. return tlb0_set_base(addr, vcpu_e500->gtlb_params[0].sets,
  199. vcpu_e500->gtlb_params[0].ways);
  200. }
  201. static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel)
  202. {
  203. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  204. int esel = get_tlb_esel_bit(vcpu);
  205. if (tlbsel == 0) {
  206. esel &= vcpu_e500->gtlb_params[0].ways - 1;
  207. esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2);
  208. } else {
  209. esel &= vcpu_e500->gtlb_params[tlbsel].entries - 1;
  210. }
  211. return esel;
  212. }
  213. /* Search the guest TLB for a matching entry. */
  214. static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
  215. gva_t eaddr, int tlbsel, unsigned int pid, int as)
  216. {
  217. int size = vcpu_e500->gtlb_params[tlbsel].entries;
  218. unsigned int set_base, offset;
  219. int i;
  220. if (tlbsel == 0) {
  221. set_base = gtlb0_set_base(vcpu_e500, eaddr);
  222. size = vcpu_e500->gtlb_params[0].ways;
  223. } else {
  224. if (eaddr < vcpu_e500->tlb1_min_eaddr ||
  225. eaddr > vcpu_e500->tlb1_max_eaddr)
  226. return -1;
  227. set_base = 0;
  228. }
  229. offset = vcpu_e500->gtlb_offset[tlbsel];
  230. for (i = 0; i < size; i++) {
  231. struct kvm_book3e_206_tlb_entry *tlbe =
  232. &vcpu_e500->gtlb_arch[offset + set_base + i];
  233. unsigned int tid;
  234. if (eaddr < get_tlb_eaddr(tlbe))
  235. continue;
  236. if (eaddr > get_tlb_end(tlbe))
  237. continue;
  238. tid = get_tlb_tid(tlbe);
  239. if (tid && (tid != pid))
  240. continue;
  241. if (!get_tlb_v(tlbe))
  242. continue;
  243. if (get_tlb_ts(tlbe) != as && as != -1)
  244. continue;
  245. return set_base + i;
  246. }
  247. return -1;
  248. }
  249. static inline void kvmppc_e500_ref_setup(struct tlbe_ref *ref,
  250. struct kvm_book3e_206_tlb_entry *gtlbe,
  251. pfn_t pfn)
  252. {
  253. ref->pfn = pfn;
  254. ref->flags = E500_TLB_VALID;
  255. if (tlbe_is_writable(gtlbe))
  256. ref->flags |= E500_TLB_DIRTY;
  257. }
  258. static inline void kvmppc_e500_ref_release(struct tlbe_ref *ref)
  259. {
  260. if (ref->flags & E500_TLB_VALID) {
  261. if (ref->flags & E500_TLB_DIRTY)
  262. kvm_release_pfn_dirty(ref->pfn);
  263. else
  264. kvm_release_pfn_clean(ref->pfn);
  265. ref->flags = 0;
  266. }
  267. }
  268. static void clear_tlb1_bitmap(struct kvmppc_vcpu_e500 *vcpu_e500)
  269. {
  270. if (vcpu_e500->g2h_tlb1_map)
  271. memset(vcpu_e500->g2h_tlb1_map, 0,
  272. sizeof(u64) * vcpu_e500->gtlb_params[1].entries);
  273. if (vcpu_e500->h2g_tlb1_rmap)
  274. memset(vcpu_e500->h2g_tlb1_rmap, 0,
  275. sizeof(unsigned int) * host_tlb_params[1].entries);
  276. }
  277. static void clear_tlb_privs(struct kvmppc_vcpu_e500 *vcpu_e500)
  278. {
  279. int tlbsel = 0;
  280. int i;
  281. for (i = 0; i < vcpu_e500->gtlb_params[tlbsel].entries; i++) {
  282. struct tlbe_ref *ref =
  283. &vcpu_e500->gtlb_priv[tlbsel][i].ref;
  284. kvmppc_e500_ref_release(ref);
  285. }
  286. }
  287. static void clear_tlb_refs(struct kvmppc_vcpu_e500 *vcpu_e500)
  288. {
  289. int stlbsel = 1;
  290. int i;
  291. kvmppc_e500_tlbil_all(vcpu_e500);
  292. for (i = 0; i < host_tlb_params[stlbsel].entries; i++) {
  293. struct tlbe_ref *ref =
  294. &vcpu_e500->tlb_refs[stlbsel][i];
  295. kvmppc_e500_ref_release(ref);
  296. }
  297. clear_tlb_privs(vcpu_e500);
  298. }
  299. static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
  300. unsigned int eaddr, int as)
  301. {
  302. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  303. unsigned int victim, tsized;
  304. int tlbsel;
  305. /* since we only have two TLBs, only lower bit is used. */
  306. tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1;
  307. victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
  308. tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f;
  309. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)
  310. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  311. vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0)
  312. | MAS1_TID(get_tlbmiss_tid(vcpu))
  313. | MAS1_TSIZE(tsized);
  314. vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN)
  315. | (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK);
  316. vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3;
  317. vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1)
  318. | (get_cur_pid(vcpu) << 16)
  319. | (as ? MAS6_SAS : 0);
  320. }
  321. /* TID must be supplied by the caller */
  322. static inline void kvmppc_e500_setup_stlbe(
  323. struct kvm_vcpu *vcpu,
  324. struct kvm_book3e_206_tlb_entry *gtlbe,
  325. int tsize, struct tlbe_ref *ref, u64 gvaddr,
  326. struct kvm_book3e_206_tlb_entry *stlbe)
  327. {
  328. pfn_t pfn = ref->pfn;
  329. u32 pr = vcpu->arch.shared->msr & MSR_PR;
  330. BUG_ON(!(ref->flags & E500_TLB_VALID));
  331. /* Force IPROT=0 for all guest mappings. */
  332. stlbe->mas1 = MAS1_TSIZE(tsize) | get_tlb_sts(gtlbe) | MAS1_VALID;
  333. stlbe->mas2 = (gvaddr & MAS2_EPN) |
  334. e500_shadow_mas2_attrib(gtlbe->mas2, pr);
  335. stlbe->mas7_3 = ((u64)pfn << PAGE_SHIFT) |
  336. e500_shadow_mas3_attrib(gtlbe->mas7_3, pr);
  337. #ifdef CONFIG_KVM_BOOKE_HV
  338. stlbe->mas8 = MAS8_TGS | vcpu->kvm->arch.lpid;
  339. #endif
  340. }
  341. static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
  342. u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
  343. int tlbsel, struct kvm_book3e_206_tlb_entry *stlbe,
  344. struct tlbe_ref *ref)
  345. {
  346. struct kvm_memory_slot *slot;
  347. unsigned long pfn, hva;
  348. int pfnmap = 0;
  349. int tsize = BOOK3E_PAGESZ_4K;
  350. /*
  351. * Translate guest physical to true physical, acquiring
  352. * a page reference if it is normal, non-reserved memory.
  353. *
  354. * gfn_to_memslot() must succeed because otherwise we wouldn't
  355. * have gotten this far. Eventually we should just pass the slot
  356. * pointer through from the first lookup.
  357. */
  358. slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
  359. hva = gfn_to_hva_memslot(slot, gfn);
  360. if (tlbsel == 1) {
  361. struct vm_area_struct *vma;
  362. down_read(&current->mm->mmap_sem);
  363. vma = find_vma(current->mm, hva);
  364. if (vma && hva >= vma->vm_start &&
  365. (vma->vm_flags & VM_PFNMAP)) {
  366. /*
  367. * This VMA is a physically contiguous region (e.g.
  368. * /dev/mem) that bypasses normal Linux page
  369. * management. Find the overlap between the
  370. * vma and the memslot.
  371. */
  372. unsigned long start, end;
  373. unsigned long slot_start, slot_end;
  374. pfnmap = 1;
  375. start = vma->vm_pgoff;
  376. end = start +
  377. ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
  378. pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT);
  379. slot_start = pfn - (gfn - slot->base_gfn);
  380. slot_end = slot_start + slot->npages;
  381. if (start < slot_start)
  382. start = slot_start;
  383. if (end > slot_end)
  384. end = slot_end;
  385. tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
  386. MAS1_TSIZE_SHIFT;
  387. /*
  388. * e500 doesn't implement the lowest tsize bit,
  389. * or 1K pages.
  390. */
  391. tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
  392. /*
  393. * Now find the largest tsize (up to what the guest
  394. * requested) that will cover gfn, stay within the
  395. * range, and for which gfn and pfn are mutually
  396. * aligned.
  397. */
  398. for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) {
  399. unsigned long gfn_start, gfn_end, tsize_pages;
  400. tsize_pages = 1 << (tsize - 2);
  401. gfn_start = gfn & ~(tsize_pages - 1);
  402. gfn_end = gfn_start + tsize_pages;
  403. if (gfn_start + pfn - gfn < start)
  404. continue;
  405. if (gfn_end + pfn - gfn > end)
  406. continue;
  407. if ((gfn & (tsize_pages - 1)) !=
  408. (pfn & (tsize_pages - 1)))
  409. continue;
  410. gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
  411. pfn &= ~(tsize_pages - 1);
  412. break;
  413. }
  414. } else if (vma && hva >= vma->vm_start &&
  415. (vma->vm_flags & VM_HUGETLB)) {
  416. unsigned long psize = vma_kernel_pagesize(vma);
  417. tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
  418. MAS1_TSIZE_SHIFT;
  419. /*
  420. * Take the largest page size that satisfies both host
  421. * and guest mapping
  422. */
  423. tsize = min(__ilog2(psize) - 10, tsize);
  424. /*
  425. * e500 doesn't implement the lowest tsize bit,
  426. * or 1K pages.
  427. */
  428. tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
  429. }
  430. up_read(&current->mm->mmap_sem);
  431. }
  432. if (likely(!pfnmap)) {
  433. unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT);
  434. pfn = gfn_to_pfn_memslot(vcpu_e500->vcpu.kvm, slot, gfn);
  435. if (is_error_pfn(pfn)) {
  436. printk(KERN_ERR "Couldn't get real page for gfn %lx!\n",
  437. (long)gfn);
  438. kvm_release_pfn_clean(pfn);
  439. return;
  440. }
  441. /* Align guest and physical address to page map boundaries */
  442. pfn &= ~(tsize_pages - 1);
  443. gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
  444. }
  445. /* Drop old ref and setup new one. */
  446. kvmppc_e500_ref_release(ref);
  447. kvmppc_e500_ref_setup(ref, gtlbe, pfn);
  448. kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize,
  449. ref, gvaddr, stlbe);
  450. /* Clear i-cache for new pages */
  451. kvmppc_mmu_flush_icache(pfn);
  452. }
  453. /* XXX only map the one-one case, for now use TLB0 */
  454. static void kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 *vcpu_e500,
  455. int esel,
  456. struct kvm_book3e_206_tlb_entry *stlbe)
  457. {
  458. struct kvm_book3e_206_tlb_entry *gtlbe;
  459. struct tlbe_ref *ref;
  460. gtlbe = get_entry(vcpu_e500, 0, esel);
  461. ref = &vcpu_e500->gtlb_priv[0][esel].ref;
  462. kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe),
  463. get_tlb_raddr(gtlbe) >> PAGE_SHIFT,
  464. gtlbe, 0, stlbe, ref);
  465. }
  466. /* Caller must ensure that the specified guest TLB entry is safe to insert into
  467. * the shadow TLB. */
  468. /* XXX for both one-one and one-to-many , for now use TLB1 */
  469. static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500,
  470. u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
  471. struct kvm_book3e_206_tlb_entry *stlbe, int esel)
  472. {
  473. struct tlbe_ref *ref;
  474. unsigned int victim;
  475. victim = vcpu_e500->host_tlb1_nv++;
  476. if (unlikely(vcpu_e500->host_tlb1_nv >= tlb1_max_shadow_size()))
  477. vcpu_e500->host_tlb1_nv = 0;
  478. ref = &vcpu_e500->tlb_refs[1][victim];
  479. kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, stlbe, ref);
  480. vcpu_e500->g2h_tlb1_map[esel] |= (u64)1 << victim;
  481. vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_BITMAP;
  482. if (vcpu_e500->h2g_tlb1_rmap[victim]) {
  483. unsigned int idx = vcpu_e500->h2g_tlb1_rmap[victim];
  484. vcpu_e500->g2h_tlb1_map[idx] &= ~(1ULL << victim);
  485. }
  486. vcpu_e500->h2g_tlb1_rmap[victim] = esel;
  487. return victim;
  488. }
  489. static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500)
  490. {
  491. int size = vcpu_e500->gtlb_params[1].entries;
  492. unsigned int offset;
  493. gva_t eaddr;
  494. int i;
  495. vcpu_e500->tlb1_min_eaddr = ~0UL;
  496. vcpu_e500->tlb1_max_eaddr = 0;
  497. offset = vcpu_e500->gtlb_offset[1];
  498. for (i = 0; i < size; i++) {
  499. struct kvm_book3e_206_tlb_entry *tlbe =
  500. &vcpu_e500->gtlb_arch[offset + i];
  501. if (!get_tlb_v(tlbe))
  502. continue;
  503. eaddr = get_tlb_eaddr(tlbe);
  504. vcpu_e500->tlb1_min_eaddr =
  505. min(vcpu_e500->tlb1_min_eaddr, eaddr);
  506. eaddr = get_tlb_end(tlbe);
  507. vcpu_e500->tlb1_max_eaddr =
  508. max(vcpu_e500->tlb1_max_eaddr, eaddr);
  509. }
  510. }
  511. static int kvmppc_need_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500,
  512. struct kvm_book3e_206_tlb_entry *gtlbe)
  513. {
  514. unsigned long start, end, size;
  515. size = get_tlb_bytes(gtlbe);
  516. start = get_tlb_eaddr(gtlbe) & ~(size - 1);
  517. end = start + size - 1;
  518. return vcpu_e500->tlb1_min_eaddr == start ||
  519. vcpu_e500->tlb1_max_eaddr == end;
  520. }
  521. /* This function is supposed to be called for a adding a new valid tlb entry */
  522. static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu,
  523. struct kvm_book3e_206_tlb_entry *gtlbe)
  524. {
  525. unsigned long start, end, size;
  526. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  527. if (!get_tlb_v(gtlbe))
  528. return;
  529. size = get_tlb_bytes(gtlbe);
  530. start = get_tlb_eaddr(gtlbe) & ~(size - 1);
  531. end = start + size - 1;
  532. vcpu_e500->tlb1_min_eaddr = min(vcpu_e500->tlb1_min_eaddr, start);
  533. vcpu_e500->tlb1_max_eaddr = max(vcpu_e500->tlb1_max_eaddr, end);
  534. }
  535. static inline int kvmppc_e500_gtlbe_invalidate(
  536. struct kvmppc_vcpu_e500 *vcpu_e500,
  537. int tlbsel, int esel)
  538. {
  539. struct kvm_book3e_206_tlb_entry *gtlbe =
  540. get_entry(vcpu_e500, tlbsel, esel);
  541. if (unlikely(get_tlb_iprot(gtlbe)))
  542. return -1;
  543. if (tlbsel == 1 && kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
  544. kvmppc_recalc_tlb1map_range(vcpu_e500);
  545. gtlbe->mas1 = 0;
  546. return 0;
  547. }
  548. int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value)
  549. {
  550. int esel;
  551. if (value & MMUCSR0_TLB0FI)
  552. for (esel = 0; esel < vcpu_e500->gtlb_params[0].entries; esel++)
  553. kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel);
  554. if (value & MMUCSR0_TLB1FI)
  555. for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++)
  556. kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel);
  557. /* Invalidate all vcpu id mappings */
  558. kvmppc_e500_tlbil_all(vcpu_e500);
  559. return EMULATE_DONE;
  560. }
  561. int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, int ra, int rb)
  562. {
  563. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  564. unsigned int ia;
  565. int esel, tlbsel;
  566. gva_t ea;
  567. ea = ((ra) ? kvmppc_get_gpr(vcpu, ra) : 0) + kvmppc_get_gpr(vcpu, rb);
  568. ia = (ea >> 2) & 0x1;
  569. /* since we only have two TLBs, only lower bit is used. */
  570. tlbsel = (ea >> 3) & 0x1;
  571. if (ia) {
  572. /* invalidate all entries */
  573. for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries;
  574. esel++)
  575. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  576. } else {
  577. ea &= 0xfffff000;
  578. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel,
  579. get_cur_pid(vcpu), -1);
  580. if (esel >= 0)
  581. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  582. }
  583. /* Invalidate all vcpu id mappings */
  584. kvmppc_e500_tlbil_all(vcpu_e500);
  585. return EMULATE_DONE;
  586. }
  587. static void tlbilx_all(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
  588. int pid, int rt)
  589. {
  590. struct kvm_book3e_206_tlb_entry *tlbe;
  591. int tid, esel;
  592. /* invalidate all entries */
  593. for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; esel++) {
  594. tlbe = get_entry(vcpu_e500, tlbsel, esel);
  595. tid = get_tlb_tid(tlbe);
  596. if (rt == 0 || tid == pid) {
  597. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  598. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  599. }
  600. }
  601. }
  602. static void tlbilx_one(struct kvmppc_vcpu_e500 *vcpu_e500, int pid,
  603. int ra, int rb)
  604. {
  605. int tlbsel, esel;
  606. gva_t ea;
  607. ea = kvmppc_get_gpr(&vcpu_e500->vcpu, rb);
  608. if (ra)
  609. ea += kvmppc_get_gpr(&vcpu_e500->vcpu, ra);
  610. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  611. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, -1);
  612. if (esel >= 0) {
  613. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  614. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  615. break;
  616. }
  617. }
  618. }
  619. int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int rt, int ra, int rb)
  620. {
  621. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  622. int pid = get_cur_spid(vcpu);
  623. if (rt == 0 || rt == 1) {
  624. tlbilx_all(vcpu_e500, 0, pid, rt);
  625. tlbilx_all(vcpu_e500, 1, pid, rt);
  626. } else if (rt == 3) {
  627. tlbilx_one(vcpu_e500, pid, ra, rb);
  628. }
  629. return EMULATE_DONE;
  630. }
  631. int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu)
  632. {
  633. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  634. int tlbsel, esel;
  635. struct kvm_book3e_206_tlb_entry *gtlbe;
  636. tlbsel = get_tlb_tlbsel(vcpu);
  637. esel = get_tlb_esel(vcpu, tlbsel);
  638. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  639. vcpu->arch.shared->mas0 &= ~MAS0_NV(~0);
  640. vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  641. vcpu->arch.shared->mas1 = gtlbe->mas1;
  642. vcpu->arch.shared->mas2 = gtlbe->mas2;
  643. vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
  644. return EMULATE_DONE;
  645. }
  646. int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, int rb)
  647. {
  648. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  649. int as = !!get_cur_sas(vcpu);
  650. unsigned int pid = get_cur_spid(vcpu);
  651. int esel, tlbsel;
  652. struct kvm_book3e_206_tlb_entry *gtlbe = NULL;
  653. gva_t ea;
  654. ea = kvmppc_get_gpr(vcpu, rb);
  655. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  656. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as);
  657. if (esel >= 0) {
  658. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  659. break;
  660. }
  661. }
  662. if (gtlbe) {
  663. esel &= vcpu_e500->gtlb_params[tlbsel].ways - 1;
  664. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel)
  665. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  666. vcpu->arch.shared->mas1 = gtlbe->mas1;
  667. vcpu->arch.shared->mas2 = gtlbe->mas2;
  668. vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
  669. } else {
  670. int victim;
  671. /* since we only have two TLBs, only lower bit is used. */
  672. tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1;
  673. victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
  674. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel)
  675. | MAS0_ESEL(victim)
  676. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  677. vcpu->arch.shared->mas1 =
  678. (vcpu->arch.shared->mas6 & MAS6_SPID0)
  679. | (vcpu->arch.shared->mas6 & (MAS6_SAS ? MAS1_TS : 0))
  680. | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0));
  681. vcpu->arch.shared->mas2 &= MAS2_EPN;
  682. vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 &
  683. MAS2_ATTRIB_MASK;
  684. vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 |
  685. MAS3_U2 | MAS3_U3;
  686. }
  687. kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS);
  688. return EMULATE_DONE;
  689. }
  690. /* sesel is for tlb1 only */
  691. static void write_stlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
  692. struct kvm_book3e_206_tlb_entry *gtlbe,
  693. struct kvm_book3e_206_tlb_entry *stlbe,
  694. int stlbsel, int sesel)
  695. {
  696. int stid;
  697. preempt_disable();
  698. stid = kvmppc_e500_get_tlb_stid(&vcpu_e500->vcpu, gtlbe);
  699. stlbe->mas1 |= MAS1_TID(stid);
  700. write_host_tlbe(vcpu_e500, stlbsel, sesel, stlbe);
  701. preempt_enable();
  702. }
  703. int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
  704. {
  705. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  706. struct kvm_book3e_206_tlb_entry *gtlbe, stlbe;
  707. int tlbsel, esel, stlbsel, sesel;
  708. int recal = 0;
  709. tlbsel = get_tlb_tlbsel(vcpu);
  710. esel = get_tlb_esel(vcpu, tlbsel);
  711. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  712. if (get_tlb_v(gtlbe)) {
  713. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  714. if ((tlbsel == 1) &&
  715. kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
  716. recal = 1;
  717. }
  718. gtlbe->mas1 = vcpu->arch.shared->mas1;
  719. gtlbe->mas2 = vcpu->arch.shared->mas2;
  720. gtlbe->mas7_3 = vcpu->arch.shared->mas7_3;
  721. trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1,
  722. gtlbe->mas2, gtlbe->mas7_3);
  723. if (tlbsel == 1) {
  724. /*
  725. * If a valid tlb1 entry is overwritten then recalculate the
  726. * min/max TLB1 map address range otherwise no need to look
  727. * in tlb1 array.
  728. */
  729. if (recal)
  730. kvmppc_recalc_tlb1map_range(vcpu_e500);
  731. else
  732. kvmppc_set_tlb1map_range(vcpu, gtlbe);
  733. }
  734. /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
  735. if (tlbe_is_host_safe(vcpu, gtlbe)) {
  736. u64 eaddr;
  737. u64 raddr;
  738. switch (tlbsel) {
  739. case 0:
  740. /* TLB0 */
  741. gtlbe->mas1 &= ~MAS1_TSIZE(~0);
  742. gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K);
  743. stlbsel = 0;
  744. kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
  745. sesel = 0; /* unused */
  746. break;
  747. case 1:
  748. /* TLB1 */
  749. eaddr = get_tlb_eaddr(gtlbe);
  750. raddr = get_tlb_raddr(gtlbe);
  751. /* Create a 4KB mapping on the host.
  752. * If the guest wanted a large page,
  753. * only the first 4KB is mapped here and the rest
  754. * are mapped on the fly. */
  755. stlbsel = 1;
  756. sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr,
  757. raddr >> PAGE_SHIFT, gtlbe, &stlbe, esel);
  758. break;
  759. default:
  760. BUG();
  761. }
  762. write_stlbe(vcpu_e500, gtlbe, &stlbe, stlbsel, sesel);
  763. }
  764. kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);
  765. return EMULATE_DONE;
  766. }
  767. static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu,
  768. gva_t eaddr, unsigned int pid, int as)
  769. {
  770. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  771. int esel, tlbsel;
  772. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  773. esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as);
  774. if (esel >= 0)
  775. return index_of(tlbsel, esel);
  776. }
  777. return -1;
  778. }
  779. /* 'linear_address' is actually an encoding of AS|PID|EADDR . */
  780. int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu,
  781. struct kvm_translation *tr)
  782. {
  783. int index;
  784. gva_t eaddr;
  785. u8 pid;
  786. u8 as;
  787. eaddr = tr->linear_address;
  788. pid = (tr->linear_address >> 32) & 0xff;
  789. as = (tr->linear_address >> 40) & 0x1;
  790. index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as);
  791. if (index < 0) {
  792. tr->valid = 0;
  793. return 0;
  794. }
  795. tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr);
  796. /* XXX what does "writeable" and "usermode" even mean? */
  797. tr->valid = 1;
  798. return 0;
  799. }
  800. int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
  801. {
  802. unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
  803. return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
  804. }
  805. int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
  806. {
  807. unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
  808. return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
  809. }
  810. void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu)
  811. {
  812. unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
  813. kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as);
  814. }
  815. void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu)
  816. {
  817. unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
  818. kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as);
  819. }
  820. gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,
  821. gva_t eaddr)
  822. {
  823. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  824. struct kvm_book3e_206_tlb_entry *gtlbe;
  825. u64 pgmask;
  826. gtlbe = get_entry(vcpu_e500, tlbsel_of(index), esel_of(index));
  827. pgmask = get_tlb_bytes(gtlbe) - 1;
  828. return get_tlb_raddr(gtlbe) | (eaddr & pgmask);
  829. }
  830. void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu)
  831. {
  832. }
  833. void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
  834. unsigned int index)
  835. {
  836. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  837. struct tlbe_priv *priv;
  838. struct kvm_book3e_206_tlb_entry *gtlbe, stlbe;
  839. int tlbsel = tlbsel_of(index);
  840. int esel = esel_of(index);
  841. int stlbsel, sesel;
  842. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  843. switch (tlbsel) {
  844. case 0:
  845. stlbsel = 0;
  846. sesel = 0; /* unused */
  847. priv = &vcpu_e500->gtlb_priv[tlbsel][esel];
  848. kvmppc_e500_setup_stlbe(vcpu, gtlbe, BOOK3E_PAGESZ_4K,
  849. &priv->ref, eaddr, &stlbe);
  850. break;
  851. case 1: {
  852. gfn_t gfn = gpaddr >> PAGE_SHIFT;
  853. stlbsel = 1;
  854. sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn,
  855. gtlbe, &stlbe, esel);
  856. break;
  857. }
  858. default:
  859. BUG();
  860. break;
  861. }
  862. write_stlbe(vcpu_e500, gtlbe, &stlbe, stlbsel, sesel);
  863. }
  864. static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500)
  865. {
  866. int i;
  867. clear_tlb1_bitmap(vcpu_e500);
  868. kfree(vcpu_e500->g2h_tlb1_map);
  869. clear_tlb_refs(vcpu_e500);
  870. kfree(vcpu_e500->gtlb_priv[0]);
  871. kfree(vcpu_e500->gtlb_priv[1]);
  872. if (vcpu_e500->shared_tlb_pages) {
  873. vfree((void *)(round_down((uintptr_t)vcpu_e500->gtlb_arch,
  874. PAGE_SIZE)));
  875. for (i = 0; i < vcpu_e500->num_shared_tlb_pages; i++) {
  876. set_page_dirty_lock(vcpu_e500->shared_tlb_pages[i]);
  877. put_page(vcpu_e500->shared_tlb_pages[i]);
  878. }
  879. vcpu_e500->num_shared_tlb_pages = 0;
  880. vcpu_e500->shared_tlb_pages = NULL;
  881. } else {
  882. kfree(vcpu_e500->gtlb_arch);
  883. }
  884. vcpu_e500->gtlb_arch = NULL;
  885. }
  886. void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  887. {
  888. sregs->u.e.mas0 = vcpu->arch.shared->mas0;
  889. sregs->u.e.mas1 = vcpu->arch.shared->mas1;
  890. sregs->u.e.mas2 = vcpu->arch.shared->mas2;
  891. sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3;
  892. sregs->u.e.mas4 = vcpu->arch.shared->mas4;
  893. sregs->u.e.mas6 = vcpu->arch.shared->mas6;
  894. sregs->u.e.mmucfg = vcpu->arch.mmucfg;
  895. sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0];
  896. sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1];
  897. sregs->u.e.tlbcfg[2] = 0;
  898. sregs->u.e.tlbcfg[3] = 0;
  899. }
  900. int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  901. {
  902. if (sregs->u.e.features & KVM_SREGS_E_ARCH206_MMU) {
  903. vcpu->arch.shared->mas0 = sregs->u.e.mas0;
  904. vcpu->arch.shared->mas1 = sregs->u.e.mas1;
  905. vcpu->arch.shared->mas2 = sregs->u.e.mas2;
  906. vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3;
  907. vcpu->arch.shared->mas4 = sregs->u.e.mas4;
  908. vcpu->arch.shared->mas6 = sregs->u.e.mas6;
  909. }
  910. return 0;
  911. }
  912. int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,
  913. struct kvm_config_tlb *cfg)
  914. {
  915. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  916. struct kvm_book3e_206_tlb_params params;
  917. char *virt;
  918. struct page **pages;
  919. struct tlbe_priv *privs[2] = {};
  920. u64 *g2h_bitmap = NULL;
  921. size_t array_len;
  922. u32 sets;
  923. int num_pages, ret, i;
  924. if (cfg->mmu_type != KVM_MMU_FSL_BOOKE_NOHV)
  925. return -EINVAL;
  926. if (copy_from_user(&params, (void __user *)(uintptr_t)cfg->params,
  927. sizeof(params)))
  928. return -EFAULT;
  929. if (params.tlb_sizes[1] > 64)
  930. return -EINVAL;
  931. if (params.tlb_ways[1] != params.tlb_sizes[1])
  932. return -EINVAL;
  933. if (params.tlb_sizes[2] != 0 || params.tlb_sizes[3] != 0)
  934. return -EINVAL;
  935. if (params.tlb_ways[2] != 0 || params.tlb_ways[3] != 0)
  936. return -EINVAL;
  937. if (!is_power_of_2(params.tlb_ways[0]))
  938. return -EINVAL;
  939. sets = params.tlb_sizes[0] >> ilog2(params.tlb_ways[0]);
  940. if (!is_power_of_2(sets))
  941. return -EINVAL;
  942. array_len = params.tlb_sizes[0] + params.tlb_sizes[1];
  943. array_len *= sizeof(struct kvm_book3e_206_tlb_entry);
  944. if (cfg->array_len < array_len)
  945. return -EINVAL;
  946. num_pages = DIV_ROUND_UP(cfg->array + array_len - 1, PAGE_SIZE) -
  947. cfg->array / PAGE_SIZE;
  948. pages = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
  949. if (!pages)
  950. return -ENOMEM;
  951. ret = get_user_pages_fast(cfg->array, num_pages, 1, pages);
  952. if (ret < 0)
  953. goto err_pages;
  954. if (ret != num_pages) {
  955. num_pages = ret;
  956. ret = -EFAULT;
  957. goto err_put_page;
  958. }
  959. virt = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL);
  960. if (!virt)
  961. goto err_put_page;
  962. privs[0] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[0],
  963. GFP_KERNEL);
  964. privs[1] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[1],
  965. GFP_KERNEL);
  966. if (!privs[0] || !privs[1])
  967. goto err_put_page;
  968. g2h_bitmap = kzalloc(sizeof(u64) * params.tlb_sizes[1],
  969. GFP_KERNEL);
  970. if (!g2h_bitmap)
  971. goto err_put_page;
  972. free_gtlb(vcpu_e500);
  973. vcpu_e500->gtlb_priv[0] = privs[0];
  974. vcpu_e500->gtlb_priv[1] = privs[1];
  975. vcpu_e500->g2h_tlb1_map = g2h_bitmap;
  976. vcpu_e500->gtlb_arch = (struct kvm_book3e_206_tlb_entry *)
  977. (virt + (cfg->array & (PAGE_SIZE - 1)));
  978. vcpu_e500->gtlb_params[0].entries = params.tlb_sizes[0];
  979. vcpu_e500->gtlb_params[1].entries = params.tlb_sizes[1];
  980. vcpu_e500->gtlb_offset[0] = 0;
  981. vcpu_e500->gtlb_offset[1] = params.tlb_sizes[0];
  982. vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE;
  983. vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  984. if (params.tlb_sizes[0] <= 2048)
  985. vcpu->arch.tlbcfg[0] |= params.tlb_sizes[0];
  986. vcpu->arch.tlbcfg[0] |= params.tlb_ways[0] << TLBnCFG_ASSOC_SHIFT;
  987. vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  988. vcpu->arch.tlbcfg[1] |= params.tlb_sizes[1];
  989. vcpu->arch.tlbcfg[1] |= params.tlb_ways[1] << TLBnCFG_ASSOC_SHIFT;
  990. vcpu_e500->shared_tlb_pages = pages;
  991. vcpu_e500->num_shared_tlb_pages = num_pages;
  992. vcpu_e500->gtlb_params[0].ways = params.tlb_ways[0];
  993. vcpu_e500->gtlb_params[0].sets = sets;
  994. vcpu_e500->gtlb_params[1].ways = params.tlb_sizes[1];
  995. vcpu_e500->gtlb_params[1].sets = 1;
  996. kvmppc_recalc_tlb1map_range(vcpu_e500);
  997. return 0;
  998. err_put_page:
  999. kfree(privs[0]);
  1000. kfree(privs[1]);
  1001. for (i = 0; i < num_pages; i++)
  1002. put_page(pages[i]);
  1003. err_pages:
  1004. kfree(pages);
  1005. return ret;
  1006. }
  1007. int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu,
  1008. struct kvm_dirty_tlb *dirty)
  1009. {
  1010. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  1011. kvmppc_recalc_tlb1map_range(vcpu_e500);
  1012. clear_tlb_refs(vcpu_e500);
  1013. return 0;
  1014. }
  1015. int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
  1016. {
  1017. struct kvm_vcpu *vcpu = &vcpu_e500->vcpu;
  1018. int entry_size = sizeof(struct kvm_book3e_206_tlb_entry);
  1019. int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE;
  1020. host_tlb_params[0].entries = mfspr(SPRN_TLB0CFG) & TLBnCFG_N_ENTRY;
  1021. host_tlb_params[1].entries = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
  1022. /*
  1023. * This should never happen on real e500 hardware, but is
  1024. * architecturally possible -- e.g. in some weird nested
  1025. * virtualization case.
  1026. */
  1027. if (host_tlb_params[0].entries == 0 ||
  1028. host_tlb_params[1].entries == 0) {
  1029. pr_err("%s: need to know host tlb size\n", __func__);
  1030. return -ENODEV;
  1031. }
  1032. host_tlb_params[0].ways = (mfspr(SPRN_TLB0CFG) & TLBnCFG_ASSOC) >>
  1033. TLBnCFG_ASSOC_SHIFT;
  1034. host_tlb_params[1].ways = host_tlb_params[1].entries;
  1035. if (!is_power_of_2(host_tlb_params[0].entries) ||
  1036. !is_power_of_2(host_tlb_params[0].ways) ||
  1037. host_tlb_params[0].entries < host_tlb_params[0].ways ||
  1038. host_tlb_params[0].ways == 0) {
  1039. pr_err("%s: bad tlb0 host config: %u entries %u ways\n",
  1040. __func__, host_tlb_params[0].entries,
  1041. host_tlb_params[0].ways);
  1042. return -ENODEV;
  1043. }
  1044. host_tlb_params[0].sets =
  1045. host_tlb_params[0].entries / host_tlb_params[0].ways;
  1046. host_tlb_params[1].sets = 1;
  1047. vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE;
  1048. vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE;
  1049. vcpu_e500->gtlb_params[0].ways = KVM_E500_TLB0_WAY_NUM;
  1050. vcpu_e500->gtlb_params[0].sets =
  1051. KVM_E500_TLB0_SIZE / KVM_E500_TLB0_WAY_NUM;
  1052. vcpu_e500->gtlb_params[1].ways = KVM_E500_TLB1_SIZE;
  1053. vcpu_e500->gtlb_params[1].sets = 1;
  1054. vcpu_e500->gtlb_arch = kmalloc(entries * entry_size, GFP_KERNEL);
  1055. if (!vcpu_e500->gtlb_arch)
  1056. return -ENOMEM;
  1057. vcpu_e500->gtlb_offset[0] = 0;
  1058. vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE;
  1059. vcpu_e500->tlb_refs[0] =
  1060. kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[0].entries,
  1061. GFP_KERNEL);
  1062. if (!vcpu_e500->tlb_refs[0])
  1063. goto err;
  1064. vcpu_e500->tlb_refs[1] =
  1065. kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[1].entries,
  1066. GFP_KERNEL);
  1067. if (!vcpu_e500->tlb_refs[1])
  1068. goto err;
  1069. vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) *
  1070. vcpu_e500->gtlb_params[0].entries,
  1071. GFP_KERNEL);
  1072. if (!vcpu_e500->gtlb_priv[0])
  1073. goto err;
  1074. vcpu_e500->gtlb_priv[1] = kzalloc(sizeof(struct tlbe_ref) *
  1075. vcpu_e500->gtlb_params[1].entries,
  1076. GFP_KERNEL);
  1077. if (!vcpu_e500->gtlb_priv[1])
  1078. goto err;
  1079. vcpu_e500->g2h_tlb1_map = kzalloc(sizeof(unsigned int) *
  1080. vcpu_e500->gtlb_params[1].entries,
  1081. GFP_KERNEL);
  1082. if (!vcpu_e500->g2h_tlb1_map)
  1083. goto err;
  1084. vcpu_e500->h2g_tlb1_rmap = kzalloc(sizeof(unsigned int) *
  1085. host_tlb_params[1].entries,
  1086. GFP_KERNEL);
  1087. if (!vcpu_e500->h2g_tlb1_rmap)
  1088. goto err;
  1089. /* Init TLB configuration register */
  1090. vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) &
  1091. ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  1092. vcpu->arch.tlbcfg[0] |= vcpu_e500->gtlb_params[0].entries;
  1093. vcpu->arch.tlbcfg[0] |=
  1094. vcpu_e500->gtlb_params[0].ways << TLBnCFG_ASSOC_SHIFT;
  1095. vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) &
  1096. ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  1097. vcpu->arch.tlbcfg[1] |= vcpu_e500->gtlb_params[1].entries;
  1098. vcpu->arch.tlbcfg[1] |=
  1099. vcpu_e500->gtlb_params[1].ways << TLBnCFG_ASSOC_SHIFT;
  1100. kvmppc_recalc_tlb1map_range(vcpu_e500);
  1101. return 0;
  1102. err:
  1103. free_gtlb(vcpu_e500);
  1104. kfree(vcpu_e500->tlb_refs[0]);
  1105. kfree(vcpu_e500->tlb_refs[1]);
  1106. return -1;
  1107. }
  1108. void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
  1109. {
  1110. free_gtlb(vcpu_e500);
  1111. kfree(vcpu_e500->h2g_tlb1_rmap);
  1112. kfree(vcpu_e500->tlb_refs[0]);
  1113. kfree(vcpu_e500->tlb_refs[1]);
  1114. }