bookehv_interrupts.S 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. /*
  2. * This program is free software; you can redistribute it and/or modify
  3. * it under the terms of the GNU General Public License, version 2, as
  4. * published by the Free Software Foundation.
  5. *
  6. * This program is distributed in the hope that it will be useful,
  7. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  8. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  9. * GNU General Public License for more details.
  10. *
  11. * You should have received a copy of the GNU General Public License
  12. * along with this program; if not, write to the Free Software
  13. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  14. *
  15. * Copyright (C) 2010-2011 Freescale Semiconductor, Inc.
  16. *
  17. * Author: Varun Sethi <varun.sethi@freescale.com>
  18. * Author: Scott Wood <scotwood@freescale.com>
  19. *
  20. * This file is derived from arch/powerpc/kvm/booke_interrupts.S
  21. */
  22. #include <asm/ppc_asm.h>
  23. #include <asm/kvm_asm.h>
  24. #include <asm/reg.h>
  25. #include <asm/mmu-44x.h>
  26. #include <asm/page.h>
  27. #include <asm/asm-compat.h>
  28. #include <asm/asm-offsets.h>
  29. #include <asm/bitsperlong.h>
  30. #include <asm/thread_info.h>
  31. #include "../kernel/head_booke.h" /* for THREAD_NORMSAVE() */
  32. #define GET_VCPU(vcpu, thread) \
  33. PPC_LL vcpu, THREAD_KVM_VCPU(thread)
  34. #define SET_VCPU(vcpu) \
  35. PPC_STL vcpu, (THREAD + THREAD_KVM_VCPU)(r2)
  36. #define LONGBYTES (BITS_PER_LONG / 8)
  37. #define VCPU_GPR(n) (VCPU_GPRS + (n * LONGBYTES))
  38. #define VCPU_GUEST_SPRG(n) (VCPU_GUEST_SPRGS + (n * LONGBYTES))
  39. /* The host stack layout: */
  40. #define HOST_R1 (0 * LONGBYTES) /* Implied by stwu. */
  41. #define HOST_CALLEE_LR (1 * LONGBYTES)
  42. #define HOST_RUN (2 * LONGBYTES) /* struct kvm_run */
  43. /*
  44. * r2 is special: it holds 'current', and it made nonvolatile in the
  45. * kernel with the -ffixed-r2 gcc option.
  46. */
  47. #define HOST_R2 (3 * LONGBYTES)
  48. #define HOST_NV_GPRS (4 * LONGBYTES)
  49. #define HOST_NV_GPR(n) (HOST_NV_GPRS + ((n - 14) * LONGBYTES))
  50. #define HOST_MIN_STACK_SIZE (HOST_NV_GPR(31) + LONGBYTES)
  51. #define HOST_STACK_SIZE ((HOST_MIN_STACK_SIZE + 15) & ~15) /* Align. */
  52. #define HOST_STACK_LR (HOST_STACK_SIZE + LONGBYTES) /* In caller stack frame. */
  53. #define NEED_EMU 0x00000001 /* emulation -- save nv regs */
  54. #define NEED_DEAR 0x00000002 /* save faulting DEAR */
  55. #define NEED_ESR 0x00000004 /* save faulting ESR */
  56. /*
  57. * On entry:
  58. * r4 = vcpu, r5 = srr0, r6 = srr1
  59. * saved in vcpu: cr, ctr, r3-r13
  60. */
  61. .macro kvm_handler_common intno, srr0, flags
  62. /* Restore host stack pointer */
  63. PPC_STL r1, VCPU_GPR(r1)(r4)
  64. PPC_STL r2, VCPU_GPR(r2)(r4)
  65. PPC_LL r1, VCPU_HOST_STACK(r4)
  66. PPC_LL r2, HOST_R2(r1)
  67. mfspr r10, SPRN_PID
  68. lwz r8, VCPU_HOST_PID(r4)
  69. PPC_LL r11, VCPU_SHARED(r4)
  70. PPC_STL r14, VCPU_GPR(r14)(r4) /* We need a non-volatile GPR. */
  71. li r14, \intno
  72. stw r10, VCPU_GUEST_PID(r4)
  73. mtspr SPRN_PID, r8
  74. #ifdef CONFIG_KVM_EXIT_TIMING
  75. /* save exit time */
  76. 1: mfspr r7, SPRN_TBRU
  77. mfspr r8, SPRN_TBRL
  78. mfspr r9, SPRN_TBRU
  79. cmpw r9, r7
  80. PPC_STL r8, VCPU_TIMING_EXIT_TBL(r4)
  81. bne- 1b
  82. PPC_STL r9, VCPU_TIMING_EXIT_TBU(r4)
  83. #endif
  84. .if \flags & NEED_EMU
  85. lwz r9, VCPU_KVM(r4)
  86. .endif
  87. oris r8, r6, MSR_CE@h
  88. #ifdef CONFIG_64BIT
  89. std r6, (VCPU_SHARED_MSR)(r11)
  90. #else
  91. stw r6, (VCPU_SHARED_MSR + 4)(r11)
  92. #endif
  93. ori r8, r8, MSR_ME | MSR_RI
  94. PPC_STL r5, VCPU_PC(r4)
  95. /*
  96. * Make sure CE/ME/RI are set (if appropriate for exception type)
  97. * whether or not the guest had it set. Since mfmsr/mtmsr are
  98. * somewhat expensive, skip in the common case where the guest
  99. * had all these bits set (and thus they're still set if
  100. * appropriate for the exception type).
  101. */
  102. cmpw r6, r8
  103. .if \flags & NEED_EMU
  104. lwz r9, KVM_LPID(r9)
  105. .endif
  106. beq 1f
  107. mfmsr r7
  108. .if \srr0 != SPRN_MCSRR0 && \srr0 != SPRN_CSRR0
  109. oris r7, r7, MSR_CE@h
  110. .endif
  111. .if \srr0 != SPRN_MCSRR0
  112. ori r7, r7, MSR_ME | MSR_RI
  113. .endif
  114. mtmsr r7
  115. 1:
  116. .if \flags & NEED_EMU
  117. /*
  118. * This assumes you have external PID support.
  119. * To support a bookehv CPU without external PID, you'll
  120. * need to look up the TLB entry and create a temporary mapping.
  121. *
  122. * FIXME: we don't currently handle if the lwepx faults. PR-mode
  123. * booke doesn't handle it either. Since Linux doesn't use
  124. * broadcast tlbivax anymore, the only way this should happen is
  125. * if the guest maps its memory execute-but-not-read, or if we
  126. * somehow take a TLB miss in the middle of this entry code and
  127. * evict the relevant entry. On e500mc, all kernel lowmem is
  128. * bolted into TLB1 large page mappings, and we don't use
  129. * broadcast invalidates, so we should not take a TLB miss here.
  130. *
  131. * Later we'll need to deal with faults here. Disallowing guest
  132. * mappings that are execute-but-not-read could be an option on
  133. * e500mc, but not on chips with an LRAT if it is used.
  134. */
  135. mfspr r3, SPRN_EPLC /* will already have correct ELPID and EGS */
  136. PPC_STL r15, VCPU_GPR(r15)(r4)
  137. PPC_STL r16, VCPU_GPR(r16)(r4)
  138. PPC_STL r17, VCPU_GPR(r17)(r4)
  139. PPC_STL r18, VCPU_GPR(r18)(r4)
  140. PPC_STL r19, VCPU_GPR(r19)(r4)
  141. mr r8, r3
  142. PPC_STL r20, VCPU_GPR(r20)(r4)
  143. rlwimi r8, r6, EPC_EAS_SHIFT - MSR_IR_LG, EPC_EAS
  144. PPC_STL r21, VCPU_GPR(r21)(r4)
  145. rlwimi r8, r6, EPC_EPR_SHIFT - MSR_PR_LG, EPC_EPR
  146. PPC_STL r22, VCPU_GPR(r22)(r4)
  147. rlwimi r8, r10, EPC_EPID_SHIFT, EPC_EPID
  148. PPC_STL r23, VCPU_GPR(r23)(r4)
  149. PPC_STL r24, VCPU_GPR(r24)(r4)
  150. PPC_STL r25, VCPU_GPR(r25)(r4)
  151. PPC_STL r26, VCPU_GPR(r26)(r4)
  152. PPC_STL r27, VCPU_GPR(r27)(r4)
  153. PPC_STL r28, VCPU_GPR(r28)(r4)
  154. PPC_STL r29, VCPU_GPR(r29)(r4)
  155. PPC_STL r30, VCPU_GPR(r30)(r4)
  156. PPC_STL r31, VCPU_GPR(r31)(r4)
  157. mtspr SPRN_EPLC, r8
  158. /* disable preemption, so we are sure we hit the fixup handler */
  159. #ifdef CONFIG_PPC64
  160. clrrdi r8,r1,THREAD_SHIFT
  161. #else
  162. rlwinm r8,r1,0,0,31-THREAD_SHIFT /* current thread_info */
  163. #endif
  164. li r7, 1
  165. stw r7, TI_PREEMPT(r8)
  166. isync
  167. /*
  168. * In case the read goes wrong, we catch it and write an invalid value
  169. * in LAST_INST instead.
  170. */
  171. 1: lwepx r9, 0, r5
  172. 2:
  173. .section .fixup, "ax"
  174. 3: li r9, KVM_INST_FETCH_FAILED
  175. b 2b
  176. .previous
  177. .section __ex_table,"a"
  178. PPC_LONG_ALIGN
  179. PPC_LONG 1b,3b
  180. .previous
  181. mtspr SPRN_EPLC, r3
  182. li r7, 0
  183. stw r7, TI_PREEMPT(r8)
  184. stw r9, VCPU_LAST_INST(r4)
  185. .endif
  186. .if \flags & NEED_ESR
  187. mfspr r8, SPRN_ESR
  188. PPC_STL r8, VCPU_FAULT_ESR(r4)
  189. .endif
  190. .if \flags & NEED_DEAR
  191. mfspr r9, SPRN_DEAR
  192. PPC_STL r9, VCPU_FAULT_DEAR(r4)
  193. .endif
  194. b kvmppc_resume_host
  195. .endm
  196. /*
  197. * For input register values, see arch/powerpc/include/asm/kvm_booke_hv_asm.h
  198. */
  199. .macro kvm_handler intno srr0, srr1, flags
  200. _GLOBAL(kvmppc_handler_\intno\()_\srr1)
  201. GET_VCPU(r11, r10)
  202. PPC_STL r3, VCPU_GPR(r3)(r11)
  203. mfspr r3, SPRN_SPRG_RSCRATCH0
  204. PPC_STL r4, VCPU_GPR(r4)(r11)
  205. PPC_LL r4, THREAD_NORMSAVE(0)(r10)
  206. PPC_STL r5, VCPU_GPR(r5)(r11)
  207. PPC_STL r13, VCPU_CR(r11)
  208. mfspr r5, \srr0
  209. PPC_STL r3, VCPU_GPR(r10)(r11)
  210. PPC_LL r3, THREAD_NORMSAVE(2)(r10)
  211. PPC_STL r6, VCPU_GPR(r6)(r11)
  212. PPC_STL r4, VCPU_GPR(r11)(r11)
  213. mfspr r6, \srr1
  214. PPC_STL r7, VCPU_GPR(r7)(r11)
  215. PPC_STL r8, VCPU_GPR(r8)(r11)
  216. PPC_STL r9, VCPU_GPR(r9)(r11)
  217. PPC_STL r3, VCPU_GPR(r13)(r11)
  218. mfctr r7
  219. PPC_STL r12, VCPU_GPR(r12)(r11)
  220. PPC_STL r7, VCPU_CTR(r11)
  221. mr r4, r11
  222. kvm_handler_common \intno, \srr0, \flags
  223. .endm
  224. .macro kvm_lvl_handler intno scratch srr0, srr1, flags
  225. _GLOBAL(kvmppc_handler_\intno\()_\srr1)
  226. mfspr r10, SPRN_SPRG_THREAD
  227. GET_VCPU(r11, r10)
  228. PPC_STL r3, VCPU_GPR(r3)(r11)
  229. mfspr r3, \scratch
  230. PPC_STL r4, VCPU_GPR(r4)(r11)
  231. PPC_LL r4, GPR9(r8)
  232. PPC_STL r5, VCPU_GPR(r5)(r11)
  233. PPC_STL r9, VCPU_CR(r11)
  234. mfspr r5, \srr0
  235. PPC_STL r3, VCPU_GPR(r8)(r11)
  236. PPC_LL r3, GPR10(r8)
  237. PPC_STL r6, VCPU_GPR(r6)(r11)
  238. PPC_STL r4, VCPU_GPR(r9)(r11)
  239. mfspr r6, \srr1
  240. PPC_LL r4, GPR11(r8)
  241. PPC_STL r7, VCPU_GPR(r7)(r11)
  242. PPC_STL r8, VCPU_GPR(r8)(r11)
  243. PPC_STL r3, VCPU_GPR(r10)(r11)
  244. mfctr r7
  245. PPC_STL r12, VCPU_GPR(r12)(r11)
  246. PPC_STL r4, VCPU_GPR(r11)(r11)
  247. PPC_STL r7, VCPU_CTR(r11)
  248. mr r4, r11
  249. kvm_handler_common \intno, \srr0, \flags
  250. .endm
  251. kvm_lvl_handler BOOKE_INTERRUPT_CRITICAL, \
  252. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  253. kvm_lvl_handler BOOKE_INTERRUPT_MACHINE_CHECK, \
  254. SPRN_SPRG_RSCRATCH_MC, SPRN_MCSRR0, SPRN_MCSRR1, 0
  255. kvm_handler BOOKE_INTERRUPT_DATA_STORAGE, \
  256. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR)
  257. kvm_handler BOOKE_INTERRUPT_INST_STORAGE, SPRN_SRR0, SPRN_SRR1, NEED_ESR
  258. kvm_handler BOOKE_INTERRUPT_EXTERNAL, SPRN_SRR0, SPRN_SRR1, 0
  259. kvm_handler BOOKE_INTERRUPT_ALIGNMENT, \
  260. SPRN_SRR0, SPRN_SRR1, (NEED_DEAR | NEED_ESR)
  261. kvm_handler BOOKE_INTERRUPT_PROGRAM, SPRN_SRR0, SPRN_SRR1, NEED_ESR
  262. kvm_handler BOOKE_INTERRUPT_FP_UNAVAIL, SPRN_SRR0, SPRN_SRR1, 0
  263. kvm_handler BOOKE_INTERRUPT_SYSCALL, SPRN_SRR0, SPRN_SRR1, 0
  264. kvm_handler BOOKE_INTERRUPT_AP_UNAVAIL, SPRN_SRR0, SPRN_SRR1, 0
  265. kvm_handler BOOKE_INTERRUPT_DECREMENTER, SPRN_SRR0, SPRN_SRR1, 0
  266. kvm_handler BOOKE_INTERRUPT_FIT, SPRN_SRR0, SPRN_SRR1, 0
  267. kvm_lvl_handler BOOKE_INTERRUPT_WATCHDOG, \
  268. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  269. kvm_handler BOOKE_INTERRUPT_DTLB_MISS, \
  270. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR)
  271. kvm_handler BOOKE_INTERRUPT_ITLB_MISS, SPRN_SRR0, SPRN_SRR1, 0
  272. kvm_handler BOOKE_INTERRUPT_SPE_UNAVAIL, SPRN_SRR0, SPRN_SRR1, 0
  273. kvm_handler BOOKE_INTERRUPT_SPE_FP_DATA, SPRN_SRR0, SPRN_SRR1, 0
  274. kvm_handler BOOKE_INTERRUPT_SPE_FP_ROUND, SPRN_SRR0, SPRN_SRR1, 0
  275. kvm_handler BOOKE_INTERRUPT_PERFORMANCE_MONITOR, SPRN_SRR0, SPRN_SRR1, 0
  276. kvm_handler BOOKE_INTERRUPT_DOORBELL, SPRN_SRR0, SPRN_SRR1, 0
  277. kvm_lvl_handler BOOKE_INTERRUPT_DOORBELL_CRITICAL, \
  278. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  279. kvm_handler BOOKE_INTERRUPT_HV_PRIV, SPRN_SRR0, SPRN_SRR1, NEED_EMU
  280. kvm_handler BOOKE_INTERRUPT_HV_SYSCALL, SPRN_SRR0, SPRN_SRR1, 0
  281. kvm_handler BOOKE_INTERRUPT_GUEST_DBELL, SPRN_GSRR0, SPRN_GSRR1, 0
  282. kvm_lvl_handler BOOKE_INTERRUPT_GUEST_DBELL_CRIT, \
  283. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  284. kvm_lvl_handler BOOKE_INTERRUPT_DEBUG, \
  285. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  286. kvm_lvl_handler BOOKE_INTERRUPT_DEBUG, \
  287. SPRN_SPRG_RSCRATCH_DBG, SPRN_DSRR0, SPRN_DSRR1, 0
  288. /* Registers:
  289. * SPRG_SCRATCH0: guest r10
  290. * r4: vcpu pointer
  291. * r11: vcpu->arch.shared
  292. * r14: KVM exit number
  293. */
  294. _GLOBAL(kvmppc_resume_host)
  295. /* Save remaining volatile guest register state to vcpu. */
  296. mfspr r3, SPRN_VRSAVE
  297. PPC_STL r0, VCPU_GPR(r0)(r4)
  298. mflr r5
  299. mfspr r6, SPRN_SPRG4
  300. PPC_STL r5, VCPU_LR(r4)
  301. mfspr r7, SPRN_SPRG5
  302. PPC_STL r3, VCPU_VRSAVE(r4)
  303. PPC_STL r6, VCPU_SHARED_SPRG4(r11)
  304. mfspr r8, SPRN_SPRG6
  305. PPC_STL r7, VCPU_SHARED_SPRG5(r11)
  306. mfspr r9, SPRN_SPRG7
  307. PPC_STL r8, VCPU_SHARED_SPRG6(r11)
  308. mfxer r3
  309. PPC_STL r9, VCPU_SHARED_SPRG7(r11)
  310. /* save guest MAS registers and restore host mas4 & mas6 */
  311. mfspr r5, SPRN_MAS0
  312. PPC_STL r3, VCPU_XER(r4)
  313. mfspr r6, SPRN_MAS1
  314. stw r5, VCPU_SHARED_MAS0(r11)
  315. mfspr r7, SPRN_MAS2
  316. stw r6, VCPU_SHARED_MAS1(r11)
  317. #ifdef CONFIG_64BIT
  318. std r7, (VCPU_SHARED_MAS2)(r11)
  319. #else
  320. stw r7, (VCPU_SHARED_MAS2 + 4)(r11)
  321. #endif
  322. mfspr r5, SPRN_MAS3
  323. mfspr r6, SPRN_MAS4
  324. stw r5, VCPU_SHARED_MAS7_3+4(r11)
  325. mfspr r7, SPRN_MAS6
  326. stw r6, VCPU_SHARED_MAS4(r11)
  327. mfspr r5, SPRN_MAS7
  328. lwz r6, VCPU_HOST_MAS4(r4)
  329. stw r7, VCPU_SHARED_MAS6(r11)
  330. lwz r8, VCPU_HOST_MAS6(r4)
  331. mtspr SPRN_MAS4, r6
  332. stw r5, VCPU_SHARED_MAS7_3+0(r11)
  333. mtspr SPRN_MAS6, r8
  334. mfspr r3, SPRN_EPCR
  335. rlwinm r3, r3, 0, ~SPRN_EPCR_DMIUH
  336. mtspr SPRN_EPCR, r3
  337. isync
  338. /* Switch to kernel stack and jump to handler. */
  339. PPC_LL r3, HOST_RUN(r1)
  340. mr r5, r14 /* intno */
  341. mr r14, r4 /* Save vcpu pointer. */
  342. bl kvmppc_handle_exit
  343. /* Restore vcpu pointer and the nonvolatiles we used. */
  344. mr r4, r14
  345. PPC_LL r14, VCPU_GPR(r14)(r4)
  346. andi. r5, r3, RESUME_FLAG_NV
  347. beq skip_nv_load
  348. PPC_LL r15, VCPU_GPR(r15)(r4)
  349. PPC_LL r16, VCPU_GPR(r16)(r4)
  350. PPC_LL r17, VCPU_GPR(r17)(r4)
  351. PPC_LL r18, VCPU_GPR(r18)(r4)
  352. PPC_LL r19, VCPU_GPR(r19)(r4)
  353. PPC_LL r20, VCPU_GPR(r20)(r4)
  354. PPC_LL r21, VCPU_GPR(r21)(r4)
  355. PPC_LL r22, VCPU_GPR(r22)(r4)
  356. PPC_LL r23, VCPU_GPR(r23)(r4)
  357. PPC_LL r24, VCPU_GPR(r24)(r4)
  358. PPC_LL r25, VCPU_GPR(r25)(r4)
  359. PPC_LL r26, VCPU_GPR(r26)(r4)
  360. PPC_LL r27, VCPU_GPR(r27)(r4)
  361. PPC_LL r28, VCPU_GPR(r28)(r4)
  362. PPC_LL r29, VCPU_GPR(r29)(r4)
  363. PPC_LL r30, VCPU_GPR(r30)(r4)
  364. PPC_LL r31, VCPU_GPR(r31)(r4)
  365. skip_nv_load:
  366. /* Should we return to the guest? */
  367. andi. r5, r3, RESUME_FLAG_HOST
  368. beq lightweight_exit
  369. srawi r3, r3, 2 /* Shift -ERR back down. */
  370. heavyweight_exit:
  371. /* Not returning to guest. */
  372. PPC_LL r5, HOST_STACK_LR(r1)
  373. /*
  374. * We already saved guest volatile register state; now save the
  375. * non-volatiles.
  376. */
  377. PPC_STL r15, VCPU_GPR(r15)(r4)
  378. PPC_STL r16, VCPU_GPR(r16)(r4)
  379. PPC_STL r17, VCPU_GPR(r17)(r4)
  380. PPC_STL r18, VCPU_GPR(r18)(r4)
  381. PPC_STL r19, VCPU_GPR(r19)(r4)
  382. PPC_STL r20, VCPU_GPR(r20)(r4)
  383. PPC_STL r21, VCPU_GPR(r21)(r4)
  384. PPC_STL r22, VCPU_GPR(r22)(r4)
  385. PPC_STL r23, VCPU_GPR(r23)(r4)
  386. PPC_STL r24, VCPU_GPR(r24)(r4)
  387. PPC_STL r25, VCPU_GPR(r25)(r4)
  388. PPC_STL r26, VCPU_GPR(r26)(r4)
  389. PPC_STL r27, VCPU_GPR(r27)(r4)
  390. PPC_STL r28, VCPU_GPR(r28)(r4)
  391. PPC_STL r29, VCPU_GPR(r29)(r4)
  392. PPC_STL r30, VCPU_GPR(r30)(r4)
  393. PPC_STL r31, VCPU_GPR(r31)(r4)
  394. /* Load host non-volatile register state from host stack. */
  395. PPC_LL r14, HOST_NV_GPR(r14)(r1)
  396. PPC_LL r15, HOST_NV_GPR(r15)(r1)
  397. PPC_LL r16, HOST_NV_GPR(r16)(r1)
  398. PPC_LL r17, HOST_NV_GPR(r17)(r1)
  399. PPC_LL r18, HOST_NV_GPR(r18)(r1)
  400. PPC_LL r19, HOST_NV_GPR(r19)(r1)
  401. PPC_LL r20, HOST_NV_GPR(r20)(r1)
  402. PPC_LL r21, HOST_NV_GPR(r21)(r1)
  403. PPC_LL r22, HOST_NV_GPR(r22)(r1)
  404. PPC_LL r23, HOST_NV_GPR(r23)(r1)
  405. PPC_LL r24, HOST_NV_GPR(r24)(r1)
  406. PPC_LL r25, HOST_NV_GPR(r25)(r1)
  407. PPC_LL r26, HOST_NV_GPR(r26)(r1)
  408. PPC_LL r27, HOST_NV_GPR(r27)(r1)
  409. PPC_LL r28, HOST_NV_GPR(r28)(r1)
  410. PPC_LL r29, HOST_NV_GPR(r29)(r1)
  411. PPC_LL r30, HOST_NV_GPR(r30)(r1)
  412. PPC_LL r31, HOST_NV_GPR(r31)(r1)
  413. /* Return to kvm_vcpu_run(). */
  414. mtlr r5
  415. addi r1, r1, HOST_STACK_SIZE
  416. /* r3 still contains the return code from kvmppc_handle_exit(). */
  417. blr
  418. /* Registers:
  419. * r3: kvm_run pointer
  420. * r4: vcpu pointer
  421. */
  422. _GLOBAL(__kvmppc_vcpu_run)
  423. stwu r1, -HOST_STACK_SIZE(r1)
  424. PPC_STL r1, VCPU_HOST_STACK(r4) /* Save stack pointer to vcpu. */
  425. /* Save host state to stack. */
  426. PPC_STL r3, HOST_RUN(r1)
  427. mflr r3
  428. PPC_STL r3, HOST_STACK_LR(r1)
  429. /* Save host non-volatile register state to stack. */
  430. PPC_STL r14, HOST_NV_GPR(r14)(r1)
  431. PPC_STL r15, HOST_NV_GPR(r15)(r1)
  432. PPC_STL r16, HOST_NV_GPR(r16)(r1)
  433. PPC_STL r17, HOST_NV_GPR(r17)(r1)
  434. PPC_STL r18, HOST_NV_GPR(r18)(r1)
  435. PPC_STL r19, HOST_NV_GPR(r19)(r1)
  436. PPC_STL r20, HOST_NV_GPR(r20)(r1)
  437. PPC_STL r21, HOST_NV_GPR(r21)(r1)
  438. PPC_STL r22, HOST_NV_GPR(r22)(r1)
  439. PPC_STL r23, HOST_NV_GPR(r23)(r1)
  440. PPC_STL r24, HOST_NV_GPR(r24)(r1)
  441. PPC_STL r25, HOST_NV_GPR(r25)(r1)
  442. PPC_STL r26, HOST_NV_GPR(r26)(r1)
  443. PPC_STL r27, HOST_NV_GPR(r27)(r1)
  444. PPC_STL r28, HOST_NV_GPR(r28)(r1)
  445. PPC_STL r29, HOST_NV_GPR(r29)(r1)
  446. PPC_STL r30, HOST_NV_GPR(r30)(r1)
  447. PPC_STL r31, HOST_NV_GPR(r31)(r1)
  448. /* Load guest non-volatiles. */
  449. PPC_LL r14, VCPU_GPR(r14)(r4)
  450. PPC_LL r15, VCPU_GPR(r15)(r4)
  451. PPC_LL r16, VCPU_GPR(r16)(r4)
  452. PPC_LL r17, VCPU_GPR(r17)(r4)
  453. PPC_LL r18, VCPU_GPR(r18)(r4)
  454. PPC_LL r19, VCPU_GPR(r19)(r4)
  455. PPC_LL r20, VCPU_GPR(r20)(r4)
  456. PPC_LL r21, VCPU_GPR(r21)(r4)
  457. PPC_LL r22, VCPU_GPR(r22)(r4)
  458. PPC_LL r23, VCPU_GPR(r23)(r4)
  459. PPC_LL r24, VCPU_GPR(r24)(r4)
  460. PPC_LL r25, VCPU_GPR(r25)(r4)
  461. PPC_LL r26, VCPU_GPR(r26)(r4)
  462. PPC_LL r27, VCPU_GPR(r27)(r4)
  463. PPC_LL r28, VCPU_GPR(r28)(r4)
  464. PPC_LL r29, VCPU_GPR(r29)(r4)
  465. PPC_LL r30, VCPU_GPR(r30)(r4)
  466. PPC_LL r31, VCPU_GPR(r31)(r4)
  467. lightweight_exit:
  468. PPC_STL r2, HOST_R2(r1)
  469. mfspr r3, SPRN_PID
  470. stw r3, VCPU_HOST_PID(r4)
  471. lwz r3, VCPU_GUEST_PID(r4)
  472. mtspr SPRN_PID, r3
  473. /* Save vcpu pointer for the exception handlers
  474. * must be done before loading guest r2.
  475. */
  476. // SET_VCPU(r4)
  477. PPC_LL r11, VCPU_SHARED(r4)
  478. /* Save host mas4 and mas6 and load guest MAS registers */
  479. mfspr r3, SPRN_MAS4
  480. stw r3, VCPU_HOST_MAS4(r4)
  481. mfspr r3, SPRN_MAS6
  482. stw r3, VCPU_HOST_MAS6(r4)
  483. lwz r3, VCPU_SHARED_MAS0(r11)
  484. lwz r5, VCPU_SHARED_MAS1(r11)
  485. #ifdef CONFIG_64BIT
  486. ld r6, (VCPU_SHARED_MAS2)(r11)
  487. #else
  488. lwz r6, (VCPU_SHARED_MAS2 + 4)(r11)
  489. #endif
  490. lwz r7, VCPU_SHARED_MAS7_3+4(r11)
  491. lwz r8, VCPU_SHARED_MAS4(r11)
  492. mtspr SPRN_MAS0, r3
  493. mtspr SPRN_MAS1, r5
  494. mtspr SPRN_MAS2, r6
  495. mtspr SPRN_MAS3, r7
  496. mtspr SPRN_MAS4, r8
  497. lwz r3, VCPU_SHARED_MAS6(r11)
  498. lwz r5, VCPU_SHARED_MAS7_3+0(r11)
  499. mtspr SPRN_MAS6, r3
  500. mtspr SPRN_MAS7, r5
  501. /* Disable MAS register updates via exception */
  502. mfspr r3, SPRN_EPCR
  503. oris r3, r3, SPRN_EPCR_DMIUH@h
  504. mtspr SPRN_EPCR, r3
  505. /*
  506. * Host interrupt handlers may have clobbered these guest-readable
  507. * SPRGs, so we need to reload them here with the guest's values.
  508. */
  509. lwz r3, VCPU_VRSAVE(r4)
  510. lwz r5, VCPU_SHARED_SPRG4(r11)
  511. mtspr SPRN_VRSAVE, r3
  512. lwz r6, VCPU_SHARED_SPRG5(r11)
  513. mtspr SPRN_SPRG4W, r5
  514. lwz r7, VCPU_SHARED_SPRG6(r11)
  515. mtspr SPRN_SPRG5W, r6
  516. lwz r8, VCPU_SHARED_SPRG7(r11)
  517. mtspr SPRN_SPRG6W, r7
  518. mtspr SPRN_SPRG7W, r8
  519. /* Load some guest volatiles. */
  520. PPC_LL r3, VCPU_LR(r4)
  521. PPC_LL r5, VCPU_XER(r4)
  522. PPC_LL r6, VCPU_CTR(r4)
  523. PPC_LL r7, VCPU_CR(r4)
  524. PPC_LL r8, VCPU_PC(r4)
  525. #ifdef CONFIG_64BIT
  526. ld r9, (VCPU_SHARED_MSR)(r11)
  527. #else
  528. lwz r9, (VCPU_SHARED_MSR + 4)(r11)
  529. #endif
  530. PPC_LL r0, VCPU_GPR(r0)(r4)
  531. PPC_LL r1, VCPU_GPR(r1)(r4)
  532. PPC_LL r2, VCPU_GPR(r2)(r4)
  533. PPC_LL r10, VCPU_GPR(r10)(r4)
  534. PPC_LL r11, VCPU_GPR(r11)(r4)
  535. PPC_LL r12, VCPU_GPR(r12)(r4)
  536. PPC_LL r13, VCPU_GPR(r13)(r4)
  537. mtlr r3
  538. mtxer r5
  539. mtctr r6
  540. mtcr r7
  541. mtsrr0 r8
  542. mtsrr1 r9
  543. #ifdef CONFIG_KVM_EXIT_TIMING
  544. /* save enter time */
  545. 1:
  546. mfspr r6, SPRN_TBRU
  547. mfspr r7, SPRN_TBRL
  548. mfspr r8, SPRN_TBRU
  549. cmpw r8, r6
  550. PPC_STL r7, VCPU_TIMING_LAST_ENTER_TBL(r4)
  551. bne 1b
  552. PPC_STL r8, VCPU_TIMING_LAST_ENTER_TBU(r4)
  553. #endif
  554. /* Finish loading guest volatiles and jump to guest. */
  555. PPC_LL r5, VCPU_GPR(r5)(r4)
  556. PPC_LL r6, VCPU_GPR(r6)(r4)
  557. PPC_LL r7, VCPU_GPR(r7)(r4)
  558. PPC_LL r8, VCPU_GPR(r8)(r4)
  559. PPC_LL r9, VCPU_GPR(r9)(r4)
  560. PPC_LL r3, VCPU_GPR(r3)(r4)
  561. PPC_LL r4, VCPU_GPR(r4)(r4)
  562. rfi