interrupts.S 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. /*
  2. * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  3. * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License, version 2, as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  17. */
  18. #include <linux/linkage.h>
  19. #include <linux/const.h>
  20. #include <asm/unified.h>
  21. #include <asm/page.h>
  22. #include <asm/ptrace.h>
  23. #include <asm/asm-offsets.h>
  24. #include <asm/kvm_asm.h>
  25. #include <asm/kvm_arm.h>
  26. #include <asm/vfpmacros.h>
  27. #include "interrupts_head.S"
  28. .text
  29. __kvm_hyp_code_start:
  30. .globl __kvm_hyp_code_start
  31. /********************************************************************
  32. * Flush per-VMID TLBs
  33. *
  34. * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
  35. *
  36. * We rely on the hardware to broadcast the TLB invalidation to all CPUs
  37. * inside the inner-shareable domain (which is the case for all v7
  38. * implementations). If we come across a non-IS SMP implementation, we'll
  39. * have to use an IPI based mechanism. Until then, we stick to the simple
  40. * hardware assisted version.
  41. *
  42. * As v7 does not support flushing per IPA, just nuke the whole TLB
  43. * instead, ignoring the ipa value.
  44. */
  45. ENTRY(__kvm_tlb_flush_vmid_ipa)
  46. push {r2, r3}
  47. add r0, r0, #KVM_VTTBR
  48. ldrd r2, r3, [r0]
  49. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  50. isb
  51. mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
  52. dsb
  53. isb
  54. mov r2, #0
  55. mov r3, #0
  56. mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
  57. isb @ Not necessary if followed by eret
  58. pop {r2, r3}
  59. bx lr
  60. ENDPROC(__kvm_tlb_flush_vmid_ipa)
  61. /********************************************************************
  62. * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
  63. * domain, for all VMIDs
  64. *
  65. * void __kvm_flush_vm_context(void);
  66. */
  67. ENTRY(__kvm_flush_vm_context)
  68. mov r0, #0 @ rn parameter for c15 flushes is SBZ
  69. /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
  70. mcr p15, 4, r0, c8, c3, 4
  71. /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
  72. mcr p15, 0, r0, c7, c1, 0
  73. dsb
  74. isb @ Not necessary if followed by eret
  75. bx lr
  76. ENDPROC(__kvm_flush_vm_context)
  77. /********************************************************************
  78. * Hypervisor world-switch code
  79. *
  80. *
  81. * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
  82. */
  83. ENTRY(__kvm_vcpu_run)
  84. @ Save the vcpu pointer
  85. mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
  86. save_host_regs
  87. restore_vgic_state
  88. restore_timer_state
  89. @ Store hardware CP15 state and load guest state
  90. read_cp15_state store_to_vcpu = 0
  91. write_cp15_state read_from_vcpu = 1
  92. @ If the host kernel has not been configured with VFPv3 support,
  93. @ then it is safer if we deny guests from using it as well.
  94. #ifdef CONFIG_VFPv3
  95. @ Set FPEXC_EN so the guest doesn't trap floating point instructions
  96. VFPFMRX r2, FPEXC @ VMRS
  97. push {r2}
  98. orr r2, r2, #FPEXC_EN
  99. VFPFMXR FPEXC, r2 @ VMSR
  100. #endif
  101. @ Configure Hyp-role
  102. configure_hyp_role vmentry
  103. @ Trap coprocessor CRx accesses
  104. set_hstr vmentry
  105. set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
  106. set_hdcr vmentry
  107. @ Write configured ID register into MIDR alias
  108. ldr r1, [vcpu, #VCPU_MIDR]
  109. mcr p15, 4, r1, c0, c0, 0
  110. @ Write guest view of MPIDR into VMPIDR
  111. ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
  112. mcr p15, 4, r1, c0, c0, 5
  113. @ Set up guest memory translation
  114. ldr r1, [vcpu, #VCPU_KVM]
  115. add r1, r1, #KVM_VTTBR
  116. ldrd r2, r3, [r1]
  117. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  118. @ We're all done, just restore the GPRs and go to the guest
  119. restore_guest_regs
  120. clrex @ Clear exclusive monitor
  121. eret
  122. __kvm_vcpu_return:
  123. /*
  124. * return convention:
  125. * guest r0, r1, r2 saved on the stack
  126. * r0: vcpu pointer
  127. * r1: exception code
  128. */
  129. save_guest_regs
  130. @ Set VMID == 0
  131. mov r2, #0
  132. mov r3, #0
  133. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  134. @ Don't trap coprocessor accesses for host kernel
  135. set_hstr vmexit
  136. set_hdcr vmexit
  137. set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
  138. #ifdef CONFIG_VFPv3
  139. @ Save floating point registers we if let guest use them.
  140. tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
  141. bne after_vfp_restore
  142. @ Switch VFP/NEON hardware state to the host's
  143. add r7, vcpu, #VCPU_VFP_GUEST
  144. store_vfp_state r7
  145. add r7, vcpu, #VCPU_VFP_HOST
  146. ldr r7, [r7]
  147. restore_vfp_state r7
  148. after_vfp_restore:
  149. @ Restore FPEXC_EN which we clobbered on entry
  150. pop {r2}
  151. VFPFMXR FPEXC, r2
  152. #endif
  153. @ Reset Hyp-role
  154. configure_hyp_role vmexit
  155. @ Let host read hardware MIDR
  156. mrc p15, 0, r2, c0, c0, 0
  157. mcr p15, 4, r2, c0, c0, 0
  158. @ Back to hardware MPIDR
  159. mrc p15, 0, r2, c0, c0, 5
  160. mcr p15, 4, r2, c0, c0, 5
  161. @ Store guest CP15 state and restore host state
  162. read_cp15_state store_to_vcpu = 1
  163. write_cp15_state read_from_vcpu = 0
  164. save_timer_state
  165. save_vgic_state
  166. restore_host_regs
  167. clrex @ Clear exclusive monitor
  168. mov r0, r1 @ Return the return code
  169. mov r1, #0 @ Clear upper bits in return value
  170. bx lr @ return to IOCTL
  171. /********************************************************************
  172. * Call function in Hyp mode
  173. *
  174. *
  175. * u64 kvm_call_hyp(void *hypfn, ...);
  176. *
  177. * This is not really a variadic function in the classic C-way and care must
  178. * be taken when calling this to ensure parameters are passed in registers
  179. * only, since the stack will change between the caller and the callee.
  180. *
  181. * Call the function with the first argument containing a pointer to the
  182. * function you wish to call in Hyp mode, and subsequent arguments will be
  183. * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
  184. * function pointer can be passed). The function being called must be mapped
  185. * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
  186. * passed in r0 and r1.
  187. *
  188. * The calling convention follows the standard AAPCS:
  189. * r0 - r3: caller save
  190. * r12: caller save
  191. * rest: callee save
  192. */
  193. ENTRY(kvm_call_hyp)
  194. hvc #0
  195. bx lr
  196. /********************************************************************
  197. * Hypervisor exception vector and handlers
  198. *
  199. *
  200. * The KVM/ARM Hypervisor ABI is defined as follows:
  201. *
  202. * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
  203. * instruction is issued since all traps are disabled when running the host
  204. * kernel as per the Hyp-mode initialization at boot time.
  205. *
  206. * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
  207. * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
  208. * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
  209. * instructions are called from within Hyp-mode.
  210. *
  211. * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
  212. * Switching to Hyp mode is done through a simple HVC #0 instruction. The
  213. * exception vector code will check that the HVC comes from VMID==0 and if
  214. * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
  215. * - r0 contains a pointer to a HYP function
  216. * - r1, r2, and r3 contain arguments to the above function.
  217. * - The HYP function will be called with its arguments in r0, r1 and r2.
  218. * On HYP function return, we return directly to SVC.
  219. *
  220. * Note that the above is used to execute code in Hyp-mode from a host-kernel
  221. * point of view, and is a different concept from performing a world-switch and
  222. * executing guest code SVC mode (with a VMID != 0).
  223. */
  224. /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
  225. .macro bad_exception exception_code, panic_str
  226. push {r0-r2}
  227. mrrc p15, 6, r0, r1, c2 @ Read VTTBR
  228. lsr r1, r1, #16
  229. ands r1, r1, #0xff
  230. beq 99f
  231. load_vcpu @ Load VCPU pointer
  232. .if \exception_code == ARM_EXCEPTION_DATA_ABORT
  233. mrc p15, 4, r2, c5, c2, 0 @ HSR
  234. mrc p15, 4, r1, c6, c0, 0 @ HDFAR
  235. str r2, [vcpu, #VCPU_HSR]
  236. str r1, [vcpu, #VCPU_HxFAR]
  237. .endif
  238. .if \exception_code == ARM_EXCEPTION_PREF_ABORT
  239. mrc p15, 4, r2, c5, c2, 0 @ HSR
  240. mrc p15, 4, r1, c6, c0, 2 @ HIFAR
  241. str r2, [vcpu, #VCPU_HSR]
  242. str r1, [vcpu, #VCPU_HxFAR]
  243. .endif
  244. mov r1, #\exception_code
  245. b __kvm_vcpu_return
  246. @ We were in the host already. Let's craft a panic-ing return to SVC.
  247. 99: mrs r2, cpsr
  248. bic r2, r2, #MODE_MASK
  249. orr r2, r2, #SVC_MODE
  250. THUMB( orr r2, r2, #PSR_T_BIT )
  251. msr spsr_cxsf, r2
  252. mrs r1, ELR_hyp
  253. ldr r2, =BSYM(panic)
  254. msr ELR_hyp, r2
  255. ldr r0, =\panic_str
  256. eret
  257. .endm
  258. .text
  259. .align 5
  260. __kvm_hyp_vector:
  261. .globl __kvm_hyp_vector
  262. @ Hyp-mode exception vector
  263. W(b) hyp_reset
  264. W(b) hyp_undef
  265. W(b) hyp_svc
  266. W(b) hyp_pabt
  267. W(b) hyp_dabt
  268. W(b) hyp_hvc
  269. W(b) hyp_irq
  270. W(b) hyp_fiq
  271. .align
  272. hyp_reset:
  273. b hyp_reset
  274. .align
  275. hyp_undef:
  276. bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
  277. .align
  278. hyp_svc:
  279. bad_exception ARM_EXCEPTION_HVC, svc_die_str
  280. .align
  281. hyp_pabt:
  282. bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
  283. .align
  284. hyp_dabt:
  285. bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
  286. .align
  287. hyp_hvc:
  288. /*
  289. * Getting here is either becuase of a trap from a guest or from calling
  290. * HVC from the host kernel, which means "switch to Hyp mode".
  291. */
  292. push {r0, r1, r2}
  293. @ Check syndrome register
  294. mrc p15, 4, r1, c5, c2, 0 @ HSR
  295. lsr r0, r1, #HSR_EC_SHIFT
  296. #ifdef CONFIG_VFPv3
  297. cmp r0, #HSR_EC_CP_0_13
  298. beq switch_to_guest_vfp
  299. #endif
  300. cmp r0, #HSR_EC_HVC
  301. bne guest_trap @ Not HVC instr.
  302. /*
  303. * Let's check if the HVC came from VMID 0 and allow simple
  304. * switch to Hyp mode
  305. */
  306. mrrc p15, 6, r0, r2, c2
  307. lsr r2, r2, #16
  308. and r2, r2, #0xff
  309. cmp r2, #0
  310. bne guest_trap @ Guest called HVC
  311. host_switch_to_hyp:
  312. pop {r0, r1, r2}
  313. push {lr}
  314. mrs lr, SPSR
  315. push {lr}
  316. mov lr, r0
  317. mov r0, r1
  318. mov r1, r2
  319. mov r2, r3
  320. THUMB( orr lr, #1)
  321. blx lr @ Call the HYP function
  322. pop {lr}
  323. msr SPSR_csxf, lr
  324. pop {lr}
  325. eret
  326. guest_trap:
  327. load_vcpu @ Load VCPU pointer to r0
  328. str r1, [vcpu, #VCPU_HSR]
  329. @ Check if we need the fault information
  330. lsr r1, r1, #HSR_EC_SHIFT
  331. cmp r1, #HSR_EC_IABT
  332. mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
  333. beq 2f
  334. cmp r1, #HSR_EC_DABT
  335. bne 1f
  336. mrc p15, 4, r2, c6, c0, 0 @ HDFAR
  337. 2: str r2, [vcpu, #VCPU_HxFAR]
  338. /*
  339. * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
  340. *
  341. * Abort on the stage 2 translation for a memory access from a
  342. * Non-secure PL1 or PL0 mode:
  343. *
  344. * For any Access flag fault or Translation fault, and also for any
  345. * Permission fault on the stage 2 translation of a memory access
  346. * made as part of a translation table walk for a stage 1 translation,
  347. * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
  348. * is UNKNOWN.
  349. */
  350. /* Check for permission fault, and S1PTW */
  351. mrc p15, 4, r1, c5, c2, 0 @ HSR
  352. and r0, r1, #HSR_FSC_TYPE
  353. cmp r0, #FSC_PERM
  354. tsteq r1, #(1 << 7) @ S1PTW
  355. mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
  356. bne 3f
  357. /* Resolve IPA using the xFAR */
  358. mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
  359. isb
  360. mrrc p15, 0, r0, r1, c7 @ PAR
  361. tst r0, #1
  362. bne 4f @ Failed translation
  363. ubfx r2, r0, #12, #20
  364. lsl r2, r2, #4
  365. orr r2, r2, r1, lsl #24
  366. 3: load_vcpu @ Load VCPU pointer to r0
  367. str r2, [r0, #VCPU_HPFAR]
  368. 1: mov r1, #ARM_EXCEPTION_HVC
  369. b __kvm_vcpu_return
  370. 4: pop {r0, r1, r2} @ Failed translation, return to guest
  371. eret
  372. /*
  373. * If VFPv3 support is not available, then we will not switch the VFP
  374. * registers; however cp10 and cp11 accesses will still trap and fallback
  375. * to the regular coprocessor emulation code, which currently will
  376. * inject an undefined exception to the guest.
  377. */
  378. #ifdef CONFIG_VFPv3
  379. switch_to_guest_vfp:
  380. load_vcpu @ Load VCPU pointer to r0
  381. push {r3-r7}
  382. @ NEON/VFP used. Turn on VFP access.
  383. set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
  384. @ Switch VFP/NEON hardware state to the guest's
  385. add r7, r0, #VCPU_VFP_HOST
  386. ldr r7, [r7]
  387. store_vfp_state r7
  388. add r7, r0, #VCPU_VFP_GUEST
  389. restore_vfp_state r7
  390. pop {r3-r7}
  391. pop {r0-r2}
  392. eret
  393. #endif
  394. .align
  395. hyp_irq:
  396. push {r0, r1, r2}
  397. mov r1, #ARM_EXCEPTION_IRQ
  398. load_vcpu @ Load VCPU pointer to r0
  399. b __kvm_vcpu_return
  400. .align
  401. hyp_fiq:
  402. b hyp_fiq
  403. .ltorg
  404. __kvm_hyp_code_end:
  405. .globl __kvm_hyp_code_end
  406. .section ".rodata"
  407. und_die_str:
  408. .ascii "unexpected undefined exception in Hyp mode at: %#08x"
  409. pabt_die_str:
  410. .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
  411. dabt_die_str:
  412. .ascii "unexpected data abort in Hyp mode at: %#08x"
  413. svc_die_str:
  414. .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"