booke.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981
  1. /*
  2. * This program is free software; you can redistribute it and/or modify
  3. * it under the terms of the GNU General Public License, version 2, as
  4. * published by the Free Software Foundation.
  5. *
  6. * This program is distributed in the hope that it will be useful,
  7. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  8. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  9. * GNU General Public License for more details.
  10. *
  11. * You should have received a copy of the GNU General Public License
  12. * along with this program; if not, write to the Free Software
  13. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  14. *
  15. * Copyright IBM Corp. 2007
  16. * Copyright 2010-2011 Freescale Semiconductor, Inc.
  17. *
  18. * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  19. * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  20. */
  21. #include <linux/errno.h>
  22. #include <linux/err.h>
  23. #include <linux/kvm_host.h>
  24. #include <linux/gfp.h>
  25. #include <linux/module.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/fs.h>
  28. #include <asm/cputable.h>
  29. #include <asm/uaccess.h>
  30. #include <asm/kvm_ppc.h>
  31. #include "timing.h"
  32. #include <asm/cacheflush.h>
  33. #include "booke.h"
  34. unsigned long kvmppc_booke_handlers;
  35. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  36. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  37. struct kvm_stats_debugfs_item debugfs_entries[] = {
  38. { "mmio", VCPU_STAT(mmio_exits) },
  39. { "dcr", VCPU_STAT(dcr_exits) },
  40. { "sig", VCPU_STAT(signal_exits) },
  41. { "itlb_r", VCPU_STAT(itlb_real_miss_exits) },
  42. { "itlb_v", VCPU_STAT(itlb_virt_miss_exits) },
  43. { "dtlb_r", VCPU_STAT(dtlb_real_miss_exits) },
  44. { "dtlb_v", VCPU_STAT(dtlb_virt_miss_exits) },
  45. { "sysc", VCPU_STAT(syscall_exits) },
  46. { "isi", VCPU_STAT(isi_exits) },
  47. { "dsi", VCPU_STAT(dsi_exits) },
  48. { "inst_emu", VCPU_STAT(emulated_inst_exits) },
  49. { "dec", VCPU_STAT(dec_exits) },
  50. { "ext_intr", VCPU_STAT(ext_intr_exits) },
  51. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  52. { NULL }
  53. };
  54. /* TODO: use vcpu_printf() */
  55. void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
  56. {
  57. int i;
  58. printk("pc: %08lx msr: %08llx\n", vcpu->arch.pc, vcpu->arch.shared->msr);
  59. printk("lr: %08lx ctr: %08lx\n", vcpu->arch.lr, vcpu->arch.ctr);
  60. printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
  61. vcpu->arch.shared->srr1);
  62. printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
  63. for (i = 0; i < 32; i += 4) {
  64. printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
  65. kvmppc_get_gpr(vcpu, i),
  66. kvmppc_get_gpr(vcpu, i+1),
  67. kvmppc_get_gpr(vcpu, i+2),
  68. kvmppc_get_gpr(vcpu, i+3));
  69. }
  70. }
  71. #ifdef CONFIG_SPE
  72. void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
  73. {
  74. preempt_disable();
  75. enable_kernel_spe();
  76. kvmppc_save_guest_spe(vcpu);
  77. vcpu->arch.shadow_msr &= ~MSR_SPE;
  78. preempt_enable();
  79. }
  80. static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
  81. {
  82. preempt_disable();
  83. enable_kernel_spe();
  84. kvmppc_load_guest_spe(vcpu);
  85. vcpu->arch.shadow_msr |= MSR_SPE;
  86. preempt_enable();
  87. }
  88. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  89. {
  90. if (vcpu->arch.shared->msr & MSR_SPE) {
  91. if (!(vcpu->arch.shadow_msr & MSR_SPE))
  92. kvmppc_vcpu_enable_spe(vcpu);
  93. } else if (vcpu->arch.shadow_msr & MSR_SPE) {
  94. kvmppc_vcpu_disable_spe(vcpu);
  95. }
  96. }
  97. #else
  98. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  99. {
  100. }
  101. #endif
  102. /*
  103. * Helper function for "full" MSR writes. No need to call this if only
  104. * EE/CE/ME/DE/RI are changing.
  105. */
  106. void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
  107. {
  108. u32 old_msr = vcpu->arch.shared->msr;
  109. vcpu->arch.shared->msr = new_msr;
  110. kvmppc_mmu_msr_notify(vcpu, old_msr);
  111. kvmppc_vcpu_sync_spe(vcpu);
  112. }
  113. static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
  114. unsigned int priority)
  115. {
  116. set_bit(priority, &vcpu->arch.pending_exceptions);
  117. }
  118. static void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
  119. ulong dear_flags, ulong esr_flags)
  120. {
  121. vcpu->arch.queued_dear = dear_flags;
  122. vcpu->arch.queued_esr = esr_flags;
  123. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
  124. }
  125. static void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu,
  126. ulong dear_flags, ulong esr_flags)
  127. {
  128. vcpu->arch.queued_dear = dear_flags;
  129. vcpu->arch.queued_esr = esr_flags;
  130. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
  131. }
  132. static void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu,
  133. ulong esr_flags)
  134. {
  135. vcpu->arch.queued_esr = esr_flags;
  136. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
  137. }
  138. void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
  139. {
  140. vcpu->arch.queued_esr = esr_flags;
  141. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
  142. }
  143. void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
  144. {
  145. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
  146. }
  147. int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
  148. {
  149. return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  150. }
  151. void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
  152. {
  153. clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  154. }
  155. void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
  156. struct kvm_interrupt *irq)
  157. {
  158. unsigned int prio = BOOKE_IRQPRIO_EXTERNAL;
  159. if (irq->irq == KVM_INTERRUPT_SET_LEVEL)
  160. prio = BOOKE_IRQPRIO_EXTERNAL_LEVEL;
  161. kvmppc_booke_queue_irqprio(vcpu, prio);
  162. }
  163. void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu,
  164. struct kvm_interrupt *irq)
  165. {
  166. clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
  167. clear_bit(BOOKE_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
  168. }
  169. /* Deliver the interrupt of the corresponding priority, if possible. */
  170. static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
  171. unsigned int priority)
  172. {
  173. int allowed = 0;
  174. ulong uninitialized_var(msr_mask);
  175. bool update_esr = false, update_dear = false;
  176. ulong crit_raw = vcpu->arch.shared->critical;
  177. ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
  178. bool crit;
  179. bool keep_irq = false;
  180. /* Truncate crit indicators in 32 bit mode */
  181. if (!(vcpu->arch.shared->msr & MSR_SF)) {
  182. crit_raw &= 0xffffffff;
  183. crit_r1 &= 0xffffffff;
  184. }
  185. /* Critical section when crit == r1 */
  186. crit = (crit_raw == crit_r1);
  187. /* ... and we're in supervisor mode */
  188. crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
  189. if (priority == BOOKE_IRQPRIO_EXTERNAL_LEVEL) {
  190. priority = BOOKE_IRQPRIO_EXTERNAL;
  191. keep_irq = true;
  192. }
  193. switch (priority) {
  194. case BOOKE_IRQPRIO_DTLB_MISS:
  195. case BOOKE_IRQPRIO_DATA_STORAGE:
  196. update_dear = true;
  197. /* fall through */
  198. case BOOKE_IRQPRIO_INST_STORAGE:
  199. case BOOKE_IRQPRIO_PROGRAM:
  200. update_esr = true;
  201. /* fall through */
  202. case BOOKE_IRQPRIO_ITLB_MISS:
  203. case BOOKE_IRQPRIO_SYSCALL:
  204. case BOOKE_IRQPRIO_FP_UNAVAIL:
  205. case BOOKE_IRQPRIO_SPE_UNAVAIL:
  206. case BOOKE_IRQPRIO_SPE_FP_DATA:
  207. case BOOKE_IRQPRIO_SPE_FP_ROUND:
  208. case BOOKE_IRQPRIO_AP_UNAVAIL:
  209. case BOOKE_IRQPRIO_ALIGNMENT:
  210. allowed = 1;
  211. msr_mask = MSR_CE|MSR_ME|MSR_DE;
  212. break;
  213. case BOOKE_IRQPRIO_CRITICAL:
  214. case BOOKE_IRQPRIO_WATCHDOG:
  215. allowed = vcpu->arch.shared->msr & MSR_CE;
  216. msr_mask = MSR_ME;
  217. break;
  218. case BOOKE_IRQPRIO_MACHINE_CHECK:
  219. allowed = vcpu->arch.shared->msr & MSR_ME;
  220. msr_mask = 0;
  221. break;
  222. case BOOKE_IRQPRIO_EXTERNAL:
  223. case BOOKE_IRQPRIO_DECREMENTER:
  224. case BOOKE_IRQPRIO_FIT:
  225. allowed = vcpu->arch.shared->msr & MSR_EE;
  226. allowed = allowed && !crit;
  227. msr_mask = MSR_CE|MSR_ME|MSR_DE;
  228. break;
  229. case BOOKE_IRQPRIO_DEBUG:
  230. allowed = vcpu->arch.shared->msr & MSR_DE;
  231. msr_mask = MSR_ME;
  232. break;
  233. }
  234. if (allowed) {
  235. vcpu->arch.shared->srr0 = vcpu->arch.pc;
  236. vcpu->arch.shared->srr1 = vcpu->arch.shared->msr;
  237. vcpu->arch.pc = vcpu->arch.ivpr | vcpu->arch.ivor[priority];
  238. if (update_esr == true)
  239. vcpu->arch.esr = vcpu->arch.queued_esr;
  240. if (update_dear == true)
  241. vcpu->arch.shared->dar = vcpu->arch.queued_dear;
  242. kvmppc_set_msr(vcpu, vcpu->arch.shared->msr & msr_mask);
  243. if (!keep_irq)
  244. clear_bit(priority, &vcpu->arch.pending_exceptions);
  245. }
  246. return allowed;
  247. }
  248. static void kvmppc_core_check_exceptions(struct kvm_vcpu *vcpu)
  249. {
  250. unsigned long *pending = &vcpu->arch.pending_exceptions;
  251. unsigned long old_pending = vcpu->arch.pending_exceptions;
  252. unsigned int priority;
  253. priority = __ffs(*pending);
  254. while (priority <= BOOKE_IRQPRIO_MAX) {
  255. if (kvmppc_booke_irqprio_deliver(vcpu, priority))
  256. break;
  257. priority = find_next_bit(pending,
  258. BITS_PER_BYTE * sizeof(*pending),
  259. priority + 1);
  260. }
  261. /* Tell the guest about our interrupt status */
  262. if (*pending)
  263. vcpu->arch.shared->int_pending = 1;
  264. else if (old_pending)
  265. vcpu->arch.shared->int_pending = 0;
  266. }
  267. /* Check pending exceptions and deliver one, if possible. */
  268. void kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu)
  269. {
  270. WARN_ON_ONCE(!irqs_disabled());
  271. kvmppc_core_check_exceptions(vcpu);
  272. if (vcpu->arch.shared->msr & MSR_WE) {
  273. local_irq_enable();
  274. kvm_vcpu_block(vcpu);
  275. local_irq_disable();
  276. kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
  277. kvmppc_core_check_exceptions(vcpu);
  278. };
  279. }
  280. int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
  281. {
  282. int ret;
  283. if (!vcpu->arch.sane) {
  284. kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  285. return -EINVAL;
  286. }
  287. local_irq_disable();
  288. kvmppc_core_prepare_to_enter(vcpu);
  289. if (signal_pending(current)) {
  290. kvm_run->exit_reason = KVM_EXIT_INTR;
  291. ret = -EINTR;
  292. goto out;
  293. }
  294. kvm_guest_enter();
  295. ret = __kvmppc_vcpu_run(kvm_run, vcpu);
  296. kvm_guest_exit();
  297. out:
  298. local_irq_enable();
  299. return ret;
  300. }
  301. /**
  302. * kvmppc_handle_exit
  303. *
  304. * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
  305. */
  306. int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
  307. unsigned int exit_nr)
  308. {
  309. enum emulation_result er;
  310. int r = RESUME_HOST;
  311. /* update before a new last_exit_type is rewritten */
  312. kvmppc_update_timing_stats(vcpu);
  313. local_irq_enable();
  314. run->exit_reason = KVM_EXIT_UNKNOWN;
  315. run->ready_for_interrupt_injection = 1;
  316. switch (exit_nr) {
  317. case BOOKE_INTERRUPT_MACHINE_CHECK:
  318. printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
  319. kvmppc_dump_vcpu(vcpu);
  320. r = RESUME_HOST;
  321. break;
  322. case BOOKE_INTERRUPT_EXTERNAL:
  323. kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
  324. if (need_resched())
  325. cond_resched();
  326. r = RESUME_GUEST;
  327. break;
  328. case BOOKE_INTERRUPT_DECREMENTER:
  329. /* Since we switched IVPR back to the host's value, the host
  330. * handled this interrupt the moment we enabled interrupts.
  331. * Now we just offer it a chance to reschedule the guest. */
  332. kvmppc_account_exit(vcpu, DEC_EXITS);
  333. if (need_resched())
  334. cond_resched();
  335. r = RESUME_GUEST;
  336. break;
  337. case BOOKE_INTERRUPT_PROGRAM:
  338. if (vcpu->arch.shared->msr & MSR_PR) {
  339. /* Program traps generated by user-level software must be handled
  340. * by the guest kernel. */
  341. kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
  342. r = RESUME_GUEST;
  343. kvmppc_account_exit(vcpu, USR_PR_INST);
  344. break;
  345. }
  346. er = kvmppc_emulate_instruction(run, vcpu);
  347. switch (er) {
  348. case EMULATE_DONE:
  349. /* don't overwrite subtypes, just account kvm_stats */
  350. kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
  351. /* Future optimization: only reload non-volatiles if
  352. * they were actually modified by emulation. */
  353. r = RESUME_GUEST_NV;
  354. break;
  355. case EMULATE_DO_DCR:
  356. run->exit_reason = KVM_EXIT_DCR;
  357. r = RESUME_HOST;
  358. break;
  359. case EMULATE_FAIL:
  360. /* XXX Deliver Program interrupt to guest. */
  361. printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
  362. __func__, vcpu->arch.pc, vcpu->arch.last_inst);
  363. /* For debugging, encode the failing instruction and
  364. * report it to userspace. */
  365. run->hw.hardware_exit_reason = ~0ULL << 32;
  366. run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
  367. r = RESUME_HOST;
  368. break;
  369. default:
  370. BUG();
  371. }
  372. break;
  373. case BOOKE_INTERRUPT_FP_UNAVAIL:
  374. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
  375. kvmppc_account_exit(vcpu, FP_UNAVAIL);
  376. r = RESUME_GUEST;
  377. break;
  378. #ifdef CONFIG_SPE
  379. case BOOKE_INTERRUPT_SPE_UNAVAIL: {
  380. if (vcpu->arch.shared->msr & MSR_SPE)
  381. kvmppc_vcpu_enable_spe(vcpu);
  382. else
  383. kvmppc_booke_queue_irqprio(vcpu,
  384. BOOKE_IRQPRIO_SPE_UNAVAIL);
  385. r = RESUME_GUEST;
  386. break;
  387. }
  388. case BOOKE_INTERRUPT_SPE_FP_DATA:
  389. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
  390. r = RESUME_GUEST;
  391. break;
  392. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  393. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
  394. r = RESUME_GUEST;
  395. break;
  396. #else
  397. case BOOKE_INTERRUPT_SPE_UNAVAIL:
  398. /*
  399. * Guest wants SPE, but host kernel doesn't support it. Send
  400. * an "unimplemented operation" program check to the guest.
  401. */
  402. kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
  403. r = RESUME_GUEST;
  404. break;
  405. /*
  406. * These really should never happen without CONFIG_SPE,
  407. * as we should never enable the real MSR[SPE] in the guest.
  408. */
  409. case BOOKE_INTERRUPT_SPE_FP_DATA:
  410. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  411. printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
  412. __func__, exit_nr, vcpu->arch.pc);
  413. run->hw.hardware_exit_reason = exit_nr;
  414. r = RESUME_HOST;
  415. break;
  416. #endif
  417. case BOOKE_INTERRUPT_DATA_STORAGE:
  418. kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
  419. vcpu->arch.fault_esr);
  420. kvmppc_account_exit(vcpu, DSI_EXITS);
  421. r = RESUME_GUEST;
  422. break;
  423. case BOOKE_INTERRUPT_INST_STORAGE:
  424. kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
  425. kvmppc_account_exit(vcpu, ISI_EXITS);
  426. r = RESUME_GUEST;
  427. break;
  428. case BOOKE_INTERRUPT_SYSCALL:
  429. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  430. (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
  431. /* KVM PV hypercalls */
  432. kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
  433. r = RESUME_GUEST;
  434. } else {
  435. /* Guest syscalls */
  436. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
  437. }
  438. kvmppc_account_exit(vcpu, SYSCALL_EXITS);
  439. r = RESUME_GUEST;
  440. break;
  441. case BOOKE_INTERRUPT_DTLB_MISS: {
  442. unsigned long eaddr = vcpu->arch.fault_dear;
  443. int gtlb_index;
  444. gpa_t gpaddr;
  445. gfn_t gfn;
  446. #ifdef CONFIG_KVM_E500
  447. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  448. (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
  449. kvmppc_map_magic(vcpu);
  450. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  451. r = RESUME_GUEST;
  452. break;
  453. }
  454. #endif
  455. /* Check the guest TLB. */
  456. gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
  457. if (gtlb_index < 0) {
  458. /* The guest didn't have a mapping for it. */
  459. kvmppc_core_queue_dtlb_miss(vcpu,
  460. vcpu->arch.fault_dear,
  461. vcpu->arch.fault_esr);
  462. kvmppc_mmu_dtlb_miss(vcpu);
  463. kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
  464. r = RESUME_GUEST;
  465. break;
  466. }
  467. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  468. gfn = gpaddr >> PAGE_SHIFT;
  469. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  470. /* The guest TLB had a mapping, but the shadow TLB
  471. * didn't, and it is RAM. This could be because:
  472. * a) the entry is mapping the host kernel, or
  473. * b) the guest used a large mapping which we're faking
  474. * Either way, we need to satisfy the fault without
  475. * invoking the guest. */
  476. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  477. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  478. r = RESUME_GUEST;
  479. } else {
  480. /* Guest has mapped and accessed a page which is not
  481. * actually RAM. */
  482. vcpu->arch.paddr_accessed = gpaddr;
  483. r = kvmppc_emulate_mmio(run, vcpu);
  484. kvmppc_account_exit(vcpu, MMIO_EXITS);
  485. }
  486. break;
  487. }
  488. case BOOKE_INTERRUPT_ITLB_MISS: {
  489. unsigned long eaddr = vcpu->arch.pc;
  490. gpa_t gpaddr;
  491. gfn_t gfn;
  492. int gtlb_index;
  493. r = RESUME_GUEST;
  494. /* Check the guest TLB. */
  495. gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
  496. if (gtlb_index < 0) {
  497. /* The guest didn't have a mapping for it. */
  498. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
  499. kvmppc_mmu_itlb_miss(vcpu);
  500. kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
  501. break;
  502. }
  503. kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
  504. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  505. gfn = gpaddr >> PAGE_SHIFT;
  506. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  507. /* The guest TLB had a mapping, but the shadow TLB
  508. * didn't. This could be because:
  509. * a) the entry is mapping the host kernel, or
  510. * b) the guest used a large mapping which we're faking
  511. * Either way, we need to satisfy the fault without
  512. * invoking the guest. */
  513. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  514. } else {
  515. /* Guest mapped and leaped at non-RAM! */
  516. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
  517. }
  518. break;
  519. }
  520. case BOOKE_INTERRUPT_DEBUG: {
  521. u32 dbsr;
  522. vcpu->arch.pc = mfspr(SPRN_CSRR0);
  523. /* clear IAC events in DBSR register */
  524. dbsr = mfspr(SPRN_DBSR);
  525. dbsr &= DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4;
  526. mtspr(SPRN_DBSR, dbsr);
  527. run->exit_reason = KVM_EXIT_DEBUG;
  528. kvmppc_account_exit(vcpu, DEBUG_EXITS);
  529. r = RESUME_HOST;
  530. break;
  531. }
  532. default:
  533. printk(KERN_EMERG "exit_nr %d\n", exit_nr);
  534. BUG();
  535. }
  536. local_irq_disable();
  537. kvmppc_core_prepare_to_enter(vcpu);
  538. if (!(r & RESUME_HOST)) {
  539. /* To avoid clobbering exit_reason, only check for signals if
  540. * we aren't already exiting to userspace for some other
  541. * reason. */
  542. if (signal_pending(current)) {
  543. run->exit_reason = KVM_EXIT_INTR;
  544. r = (-EINTR << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
  545. kvmppc_account_exit(vcpu, SIGNAL_EXITS);
  546. }
  547. }
  548. return r;
  549. }
  550. /* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
  551. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  552. {
  553. int i;
  554. int r;
  555. vcpu->arch.pc = 0;
  556. vcpu->arch.shared->msr = 0;
  557. vcpu->arch.shadow_msr = MSR_USER | MSR_DE | MSR_IS | MSR_DS;
  558. kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
  559. vcpu->arch.shadow_pid = 1;
  560. /* Eye-catching numbers so we know if the guest takes an interrupt
  561. * before it's programmed its own IVPR/IVORs. */
  562. vcpu->arch.ivpr = 0x55550000;
  563. for (i = 0; i < BOOKE_IRQPRIO_MAX; i++)
  564. vcpu->arch.ivor[i] = 0x7700 | i * 4;
  565. kvmppc_init_timing_stats(vcpu);
  566. r = kvmppc_core_vcpu_setup(vcpu);
  567. kvmppc_sanity_check(vcpu);
  568. return r;
  569. }
  570. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  571. {
  572. int i;
  573. regs->pc = vcpu->arch.pc;
  574. regs->cr = kvmppc_get_cr(vcpu);
  575. regs->ctr = vcpu->arch.ctr;
  576. regs->lr = vcpu->arch.lr;
  577. regs->xer = kvmppc_get_xer(vcpu);
  578. regs->msr = vcpu->arch.shared->msr;
  579. regs->srr0 = vcpu->arch.shared->srr0;
  580. regs->srr1 = vcpu->arch.shared->srr1;
  581. regs->pid = vcpu->arch.pid;
  582. regs->sprg0 = vcpu->arch.shared->sprg0;
  583. regs->sprg1 = vcpu->arch.shared->sprg1;
  584. regs->sprg2 = vcpu->arch.shared->sprg2;
  585. regs->sprg3 = vcpu->arch.shared->sprg3;
  586. regs->sprg4 = vcpu->arch.sprg4;
  587. regs->sprg5 = vcpu->arch.sprg5;
  588. regs->sprg6 = vcpu->arch.sprg6;
  589. regs->sprg7 = vcpu->arch.sprg7;
  590. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  591. regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
  592. return 0;
  593. }
  594. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  595. {
  596. int i;
  597. vcpu->arch.pc = regs->pc;
  598. kvmppc_set_cr(vcpu, regs->cr);
  599. vcpu->arch.ctr = regs->ctr;
  600. vcpu->arch.lr = regs->lr;
  601. kvmppc_set_xer(vcpu, regs->xer);
  602. kvmppc_set_msr(vcpu, regs->msr);
  603. vcpu->arch.shared->srr0 = regs->srr0;
  604. vcpu->arch.shared->srr1 = regs->srr1;
  605. kvmppc_set_pid(vcpu, regs->pid);
  606. vcpu->arch.shared->sprg0 = regs->sprg0;
  607. vcpu->arch.shared->sprg1 = regs->sprg1;
  608. vcpu->arch.shared->sprg2 = regs->sprg2;
  609. vcpu->arch.shared->sprg3 = regs->sprg3;
  610. vcpu->arch.sprg4 = regs->sprg4;
  611. vcpu->arch.sprg5 = regs->sprg5;
  612. vcpu->arch.sprg6 = regs->sprg6;
  613. vcpu->arch.sprg7 = regs->sprg7;
  614. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  615. kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
  616. return 0;
  617. }
  618. static void get_sregs_base(struct kvm_vcpu *vcpu,
  619. struct kvm_sregs *sregs)
  620. {
  621. u64 tb = get_tb();
  622. sregs->u.e.features |= KVM_SREGS_E_BASE;
  623. sregs->u.e.csrr0 = vcpu->arch.csrr0;
  624. sregs->u.e.csrr1 = vcpu->arch.csrr1;
  625. sregs->u.e.mcsr = vcpu->arch.mcsr;
  626. sregs->u.e.esr = vcpu->arch.esr;
  627. sregs->u.e.dear = vcpu->arch.shared->dar;
  628. sregs->u.e.tsr = vcpu->arch.tsr;
  629. sregs->u.e.tcr = vcpu->arch.tcr;
  630. sregs->u.e.dec = kvmppc_get_dec(vcpu, tb);
  631. sregs->u.e.tb = tb;
  632. sregs->u.e.vrsave = vcpu->arch.vrsave;
  633. }
  634. static int set_sregs_base(struct kvm_vcpu *vcpu,
  635. struct kvm_sregs *sregs)
  636. {
  637. if (!(sregs->u.e.features & KVM_SREGS_E_BASE))
  638. return 0;
  639. vcpu->arch.csrr0 = sregs->u.e.csrr0;
  640. vcpu->arch.csrr1 = sregs->u.e.csrr1;
  641. vcpu->arch.mcsr = sregs->u.e.mcsr;
  642. vcpu->arch.esr = sregs->u.e.esr;
  643. vcpu->arch.shared->dar = sregs->u.e.dear;
  644. vcpu->arch.vrsave = sregs->u.e.vrsave;
  645. vcpu->arch.tcr = sregs->u.e.tcr;
  646. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_DEC)
  647. vcpu->arch.dec = sregs->u.e.dec;
  648. kvmppc_emulate_dec(vcpu);
  649. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_TSR) {
  650. /*
  651. * FIXME: existing KVM timer handling is incomplete.
  652. * TSR cannot be read by the guest, and its value in
  653. * vcpu->arch is always zero. For now, just handle
  654. * the case where the caller is trying to inject a
  655. * decrementer interrupt.
  656. */
  657. if ((sregs->u.e.tsr & TSR_DIS) &&
  658. (vcpu->arch.tcr & TCR_DIE))
  659. kvmppc_core_queue_dec(vcpu);
  660. }
  661. return 0;
  662. }
  663. static void get_sregs_arch206(struct kvm_vcpu *vcpu,
  664. struct kvm_sregs *sregs)
  665. {
  666. sregs->u.e.features |= KVM_SREGS_E_ARCH206;
  667. sregs->u.e.pir = vcpu->vcpu_id;
  668. sregs->u.e.mcsrr0 = vcpu->arch.mcsrr0;
  669. sregs->u.e.mcsrr1 = vcpu->arch.mcsrr1;
  670. sregs->u.e.decar = vcpu->arch.decar;
  671. sregs->u.e.ivpr = vcpu->arch.ivpr;
  672. }
  673. static int set_sregs_arch206(struct kvm_vcpu *vcpu,
  674. struct kvm_sregs *sregs)
  675. {
  676. if (!(sregs->u.e.features & KVM_SREGS_E_ARCH206))
  677. return 0;
  678. if (sregs->u.e.pir != vcpu->vcpu_id)
  679. return -EINVAL;
  680. vcpu->arch.mcsrr0 = sregs->u.e.mcsrr0;
  681. vcpu->arch.mcsrr1 = sregs->u.e.mcsrr1;
  682. vcpu->arch.decar = sregs->u.e.decar;
  683. vcpu->arch.ivpr = sregs->u.e.ivpr;
  684. return 0;
  685. }
  686. void kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  687. {
  688. sregs->u.e.features |= KVM_SREGS_E_IVOR;
  689. sregs->u.e.ivor_low[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL];
  690. sregs->u.e.ivor_low[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK];
  691. sregs->u.e.ivor_low[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE];
  692. sregs->u.e.ivor_low[3] = vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE];
  693. sregs->u.e.ivor_low[4] = vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL];
  694. sregs->u.e.ivor_low[5] = vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT];
  695. sregs->u.e.ivor_low[6] = vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM];
  696. sregs->u.e.ivor_low[7] = vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL];
  697. sregs->u.e.ivor_low[8] = vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL];
  698. sregs->u.e.ivor_low[9] = vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL];
  699. sregs->u.e.ivor_low[10] = vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER];
  700. sregs->u.e.ivor_low[11] = vcpu->arch.ivor[BOOKE_IRQPRIO_FIT];
  701. sregs->u.e.ivor_low[12] = vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG];
  702. sregs->u.e.ivor_low[13] = vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS];
  703. sregs->u.e.ivor_low[14] = vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS];
  704. sregs->u.e.ivor_low[15] = vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG];
  705. }
  706. int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  707. {
  708. if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
  709. return 0;
  710. vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL] = sregs->u.e.ivor_low[0];
  711. vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK] = sregs->u.e.ivor_low[1];
  712. vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE] = sregs->u.e.ivor_low[2];
  713. vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE] = sregs->u.e.ivor_low[3];
  714. vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL] = sregs->u.e.ivor_low[4];
  715. vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT] = sregs->u.e.ivor_low[5];
  716. vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM] = sregs->u.e.ivor_low[6];
  717. vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL] = sregs->u.e.ivor_low[7];
  718. vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL] = sregs->u.e.ivor_low[8];
  719. vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL] = sregs->u.e.ivor_low[9];
  720. vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER] = sregs->u.e.ivor_low[10];
  721. vcpu->arch.ivor[BOOKE_IRQPRIO_FIT] = sregs->u.e.ivor_low[11];
  722. vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG] = sregs->u.e.ivor_low[12];
  723. vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS] = sregs->u.e.ivor_low[13];
  724. vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS] = sregs->u.e.ivor_low[14];
  725. vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG] = sregs->u.e.ivor_low[15];
  726. return 0;
  727. }
  728. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  729. struct kvm_sregs *sregs)
  730. {
  731. sregs->pvr = vcpu->arch.pvr;
  732. get_sregs_base(vcpu, sregs);
  733. get_sregs_arch206(vcpu, sregs);
  734. kvmppc_core_get_sregs(vcpu, sregs);
  735. return 0;
  736. }
  737. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  738. struct kvm_sregs *sregs)
  739. {
  740. int ret;
  741. if (vcpu->arch.pvr != sregs->pvr)
  742. return -EINVAL;
  743. ret = set_sregs_base(vcpu, sregs);
  744. if (ret < 0)
  745. return ret;
  746. ret = set_sregs_arch206(vcpu, sregs);
  747. if (ret < 0)
  748. return ret;
  749. return kvmppc_core_set_sregs(vcpu, sregs);
  750. }
  751. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  752. {
  753. return -ENOTSUPP;
  754. }
  755. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  756. {
  757. return -ENOTSUPP;
  758. }
  759. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  760. struct kvm_translation *tr)
  761. {
  762. int r;
  763. r = kvmppc_core_vcpu_translate(vcpu, tr);
  764. return r;
  765. }
  766. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
  767. {
  768. return -ENOTSUPP;
  769. }
  770. int kvmppc_core_prepare_memory_region(struct kvm *kvm,
  771. struct kvm_userspace_memory_region *mem)
  772. {
  773. return 0;
  774. }
  775. void kvmppc_core_commit_memory_region(struct kvm *kvm,
  776. struct kvm_userspace_memory_region *mem)
  777. {
  778. }
  779. int kvmppc_core_init_vm(struct kvm *kvm)
  780. {
  781. return 0;
  782. }
  783. void kvmppc_core_destroy_vm(struct kvm *kvm)
  784. {
  785. }
  786. int __init kvmppc_booke_init(void)
  787. {
  788. unsigned long ivor[16];
  789. unsigned long max_ivor = 0;
  790. int i;
  791. /* We install our own exception handlers by hijacking IVPR. IVPR must
  792. * be 16-bit aligned, so we need a 64KB allocation. */
  793. kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
  794. VCPU_SIZE_ORDER);
  795. if (!kvmppc_booke_handlers)
  796. return -ENOMEM;
  797. /* XXX make sure our handlers are smaller than Linux's */
  798. /* Copy our interrupt handlers to match host IVORs. That way we don't
  799. * have to swap the IVORs on every guest/host transition. */
  800. ivor[0] = mfspr(SPRN_IVOR0);
  801. ivor[1] = mfspr(SPRN_IVOR1);
  802. ivor[2] = mfspr(SPRN_IVOR2);
  803. ivor[3] = mfspr(SPRN_IVOR3);
  804. ivor[4] = mfspr(SPRN_IVOR4);
  805. ivor[5] = mfspr(SPRN_IVOR5);
  806. ivor[6] = mfspr(SPRN_IVOR6);
  807. ivor[7] = mfspr(SPRN_IVOR7);
  808. ivor[8] = mfspr(SPRN_IVOR8);
  809. ivor[9] = mfspr(SPRN_IVOR9);
  810. ivor[10] = mfspr(SPRN_IVOR10);
  811. ivor[11] = mfspr(SPRN_IVOR11);
  812. ivor[12] = mfspr(SPRN_IVOR12);
  813. ivor[13] = mfspr(SPRN_IVOR13);
  814. ivor[14] = mfspr(SPRN_IVOR14);
  815. ivor[15] = mfspr(SPRN_IVOR15);
  816. for (i = 0; i < 16; i++) {
  817. if (ivor[i] > max_ivor)
  818. max_ivor = ivor[i];
  819. memcpy((void *)kvmppc_booke_handlers + ivor[i],
  820. kvmppc_handlers_start + i * kvmppc_handler_len,
  821. kvmppc_handler_len);
  822. }
  823. flush_icache_range(kvmppc_booke_handlers,
  824. kvmppc_booke_handlers + max_ivor + kvmppc_handler_len);
  825. return 0;
  826. }
  827. void __exit kvmppc_booke_exit(void)
  828. {
  829. free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
  830. kvm_exit();
  831. }