interrupts.S 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501
  1. /*
  2. * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  3. * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License, version 2, as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  17. */
  18. #include <linux/linkage.h>
  19. #include <linux/const.h>
  20. #include <asm/unified.h>
  21. #include <asm/page.h>
  22. #include <asm/ptrace.h>
  23. #include <asm/asm-offsets.h>
  24. #include <asm/kvm_asm.h>
  25. #include <asm/kvm_arm.h>
  26. #include <asm/vfpmacros.h>
  27. #include "interrupts_head.S"
  28. .text
  29. __kvm_hyp_code_start:
  30. .globl __kvm_hyp_code_start
  31. /********************************************************************
  32. * Flush per-VMID TLBs
  33. *
  34. * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
  35. *
  36. * We rely on the hardware to broadcast the TLB invalidation to all CPUs
  37. * inside the inner-shareable domain (which is the case for all v7
  38. * implementations). If we come across a non-IS SMP implementation, we'll
  39. * have to use an IPI based mechanism. Until then, we stick to the simple
  40. * hardware assisted version.
  41. *
  42. * As v7 does not support flushing per IPA, just nuke the whole TLB
  43. * instead, ignoring the ipa value.
  44. */
  45. ENTRY(__kvm_tlb_flush_vmid_ipa)
  46. push {r2, r3}
  47. dsb ishst
  48. add r0, r0, #KVM_VTTBR
  49. ldrd r2, r3, [r0]
  50. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  51. isb
  52. mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
  53. dsb
  54. isb
  55. mov r2, #0
  56. mov r3, #0
  57. mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
  58. isb @ Not necessary if followed by eret
  59. pop {r2, r3}
  60. bx lr
  61. ENDPROC(__kvm_tlb_flush_vmid_ipa)
  62. /********************************************************************
  63. * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
  64. * domain, for all VMIDs
  65. *
  66. * void __kvm_flush_vm_context(void);
  67. */
  68. ENTRY(__kvm_flush_vm_context)
  69. mov r0, #0 @ rn parameter for c15 flushes is SBZ
  70. /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
  71. mcr p15, 4, r0, c8, c3, 4
  72. /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
  73. mcr p15, 0, r0, c7, c1, 0
  74. dsb
  75. isb @ Not necessary if followed by eret
  76. bx lr
  77. ENDPROC(__kvm_flush_vm_context)
  78. /********************************************************************
  79. * Hypervisor world-switch code
  80. *
  81. *
  82. * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
  83. */
  84. ENTRY(__kvm_vcpu_run)
  85. @ Save the vcpu pointer
  86. mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
  87. save_host_regs
  88. restore_vgic_state
  89. restore_timer_state
  90. @ Store hardware CP15 state and load guest state
  91. read_cp15_state store_to_vcpu = 0
  92. write_cp15_state read_from_vcpu = 1
  93. @ If the host kernel has not been configured with VFPv3 support,
  94. @ then it is safer if we deny guests from using it as well.
  95. #ifdef CONFIG_VFPv3
  96. @ Set FPEXC_EN so the guest doesn't trap floating point instructions
  97. VFPFMRX r2, FPEXC @ VMRS
  98. push {r2}
  99. orr r2, r2, #FPEXC_EN
  100. VFPFMXR FPEXC, r2 @ VMSR
  101. #endif
  102. @ Configure Hyp-role
  103. configure_hyp_role vmentry
  104. @ Trap coprocessor CRx accesses
  105. set_hstr vmentry
  106. set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
  107. set_hdcr vmentry
  108. @ Write configured ID register into MIDR alias
  109. ldr r1, [vcpu, #VCPU_MIDR]
  110. mcr p15, 4, r1, c0, c0, 0
  111. @ Write guest view of MPIDR into VMPIDR
  112. ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
  113. mcr p15, 4, r1, c0, c0, 5
  114. @ Set up guest memory translation
  115. ldr r1, [vcpu, #VCPU_KVM]
  116. add r1, r1, #KVM_VTTBR
  117. ldrd r2, r3, [r1]
  118. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  119. @ We're all done, just restore the GPRs and go to the guest
  120. restore_guest_regs
  121. clrex @ Clear exclusive monitor
  122. eret
  123. __kvm_vcpu_return:
  124. /*
  125. * return convention:
  126. * guest r0, r1, r2 saved on the stack
  127. * r0: vcpu pointer
  128. * r1: exception code
  129. */
  130. save_guest_regs
  131. @ Set VMID == 0
  132. mov r2, #0
  133. mov r3, #0
  134. mcrr p15, 6, r2, r3, c2 @ Write VTTBR
  135. @ Don't trap coprocessor accesses for host kernel
  136. set_hstr vmexit
  137. set_hdcr vmexit
  138. set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
  139. #ifdef CONFIG_VFPv3
  140. @ Save floating point registers we if let guest use them.
  141. tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
  142. bne after_vfp_restore
  143. @ Switch VFP/NEON hardware state to the host's
  144. add r7, vcpu, #VCPU_VFP_GUEST
  145. store_vfp_state r7
  146. add r7, vcpu, #VCPU_VFP_HOST
  147. ldr r7, [r7]
  148. restore_vfp_state r7
  149. after_vfp_restore:
  150. @ Restore FPEXC_EN which we clobbered on entry
  151. pop {r2}
  152. VFPFMXR FPEXC, r2
  153. #endif
  154. @ Reset Hyp-role
  155. configure_hyp_role vmexit
  156. @ Let host read hardware MIDR
  157. mrc p15, 0, r2, c0, c0, 0
  158. mcr p15, 4, r2, c0, c0, 0
  159. @ Back to hardware MPIDR
  160. mrc p15, 0, r2, c0, c0, 5
  161. mcr p15, 4, r2, c0, c0, 5
  162. @ Store guest CP15 state and restore host state
  163. read_cp15_state store_to_vcpu = 1
  164. write_cp15_state read_from_vcpu = 0
  165. save_timer_state
  166. save_vgic_state
  167. restore_host_regs
  168. clrex @ Clear exclusive monitor
  169. mov r0, r1 @ Return the return code
  170. mov r1, #0 @ Clear upper bits in return value
  171. bx lr @ return to IOCTL
  172. /********************************************************************
  173. * Call function in Hyp mode
  174. *
  175. *
  176. * u64 kvm_call_hyp(void *hypfn, ...);
  177. *
  178. * This is not really a variadic function in the classic C-way and care must
  179. * be taken when calling this to ensure parameters are passed in registers
  180. * only, since the stack will change between the caller and the callee.
  181. *
  182. * Call the function with the first argument containing a pointer to the
  183. * function you wish to call in Hyp mode, and subsequent arguments will be
  184. * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
  185. * function pointer can be passed). The function being called must be mapped
  186. * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
  187. * passed in r0 and r1.
  188. *
  189. * The calling convention follows the standard AAPCS:
  190. * r0 - r3: caller save
  191. * r12: caller save
  192. * rest: callee save
  193. */
  194. ENTRY(kvm_call_hyp)
  195. hvc #0
  196. bx lr
  197. /********************************************************************
  198. * Hypervisor exception vector and handlers
  199. *
  200. *
  201. * The KVM/ARM Hypervisor ABI is defined as follows:
  202. *
  203. * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
  204. * instruction is issued since all traps are disabled when running the host
  205. * kernel as per the Hyp-mode initialization at boot time.
  206. *
  207. * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
  208. * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
  209. * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
  210. * instructions are called from within Hyp-mode.
  211. *
  212. * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
  213. * Switching to Hyp mode is done through a simple HVC #0 instruction. The
  214. * exception vector code will check that the HVC comes from VMID==0 and if
  215. * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
  216. * - r0 contains a pointer to a HYP function
  217. * - r1, r2, and r3 contain arguments to the above function.
  218. * - The HYP function will be called with its arguments in r0, r1 and r2.
  219. * On HYP function return, we return directly to SVC.
  220. *
  221. * Note that the above is used to execute code in Hyp-mode from a host-kernel
  222. * point of view, and is a different concept from performing a world-switch and
  223. * executing guest code SVC mode (with a VMID != 0).
  224. */
  225. /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
  226. .macro bad_exception exception_code, panic_str
  227. push {r0-r2}
  228. mrrc p15, 6, r0, r1, c2 @ Read VTTBR
  229. lsr r1, r1, #16
  230. ands r1, r1, #0xff
  231. beq 99f
  232. load_vcpu @ Load VCPU pointer
  233. .if \exception_code == ARM_EXCEPTION_DATA_ABORT
  234. mrc p15, 4, r2, c5, c2, 0 @ HSR
  235. mrc p15, 4, r1, c6, c0, 0 @ HDFAR
  236. str r2, [vcpu, #VCPU_HSR]
  237. str r1, [vcpu, #VCPU_HxFAR]
  238. .endif
  239. .if \exception_code == ARM_EXCEPTION_PREF_ABORT
  240. mrc p15, 4, r2, c5, c2, 0 @ HSR
  241. mrc p15, 4, r1, c6, c0, 2 @ HIFAR
  242. str r2, [vcpu, #VCPU_HSR]
  243. str r1, [vcpu, #VCPU_HxFAR]
  244. .endif
  245. mov r1, #\exception_code
  246. b __kvm_vcpu_return
  247. @ We were in the host already. Let's craft a panic-ing return to SVC.
  248. 99: mrs r2, cpsr
  249. bic r2, r2, #MODE_MASK
  250. orr r2, r2, #SVC_MODE
  251. THUMB( orr r2, r2, #PSR_T_BIT )
  252. msr spsr_cxsf, r2
  253. mrs r1, ELR_hyp
  254. ldr r2, =BSYM(panic)
  255. msr ELR_hyp, r2
  256. ldr r0, =\panic_str
  257. clrex @ Clear exclusive monitor
  258. eret
  259. .endm
  260. .text
  261. .align 5
  262. __kvm_hyp_vector:
  263. .globl __kvm_hyp_vector
  264. @ Hyp-mode exception vector
  265. W(b) hyp_reset
  266. W(b) hyp_undef
  267. W(b) hyp_svc
  268. W(b) hyp_pabt
  269. W(b) hyp_dabt
  270. W(b) hyp_hvc
  271. W(b) hyp_irq
  272. W(b) hyp_fiq
  273. .align
  274. hyp_reset:
  275. b hyp_reset
  276. .align
  277. hyp_undef:
  278. bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
  279. .align
  280. hyp_svc:
  281. bad_exception ARM_EXCEPTION_HVC, svc_die_str
  282. .align
  283. hyp_pabt:
  284. bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
  285. .align
  286. hyp_dabt:
  287. bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
  288. .align
  289. hyp_hvc:
  290. /*
  291. * Getting here is either becuase of a trap from a guest or from calling
  292. * HVC from the host kernel, which means "switch to Hyp mode".
  293. */
  294. push {r0, r1, r2}
  295. @ Check syndrome register
  296. mrc p15, 4, r1, c5, c2, 0 @ HSR
  297. lsr r0, r1, #HSR_EC_SHIFT
  298. #ifdef CONFIG_VFPv3
  299. cmp r0, #HSR_EC_CP_0_13
  300. beq switch_to_guest_vfp
  301. #endif
  302. cmp r0, #HSR_EC_HVC
  303. bne guest_trap @ Not HVC instr.
  304. /*
  305. * Let's check if the HVC came from VMID 0 and allow simple
  306. * switch to Hyp mode
  307. */
  308. mrrc p15, 6, r0, r2, c2
  309. lsr r2, r2, #16
  310. and r2, r2, #0xff
  311. cmp r2, #0
  312. bne guest_trap @ Guest called HVC
  313. host_switch_to_hyp:
  314. pop {r0, r1, r2}
  315. push {lr}
  316. mrs lr, SPSR
  317. push {lr}
  318. mov lr, r0
  319. mov r0, r1
  320. mov r1, r2
  321. mov r2, r3
  322. THUMB( orr lr, #1)
  323. blx lr @ Call the HYP function
  324. pop {lr}
  325. msr SPSR_csxf, lr
  326. pop {lr}
  327. eret
  328. guest_trap:
  329. load_vcpu @ Load VCPU pointer to r0
  330. str r1, [vcpu, #VCPU_HSR]
  331. @ Check if we need the fault information
  332. lsr r1, r1, #HSR_EC_SHIFT
  333. cmp r1, #HSR_EC_IABT
  334. mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
  335. beq 2f
  336. cmp r1, #HSR_EC_DABT
  337. bne 1f
  338. mrc p15, 4, r2, c6, c0, 0 @ HDFAR
  339. 2: str r2, [vcpu, #VCPU_HxFAR]
  340. /*
  341. * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
  342. *
  343. * Abort on the stage 2 translation for a memory access from a
  344. * Non-secure PL1 or PL0 mode:
  345. *
  346. * For any Access flag fault or Translation fault, and also for any
  347. * Permission fault on the stage 2 translation of a memory access
  348. * made as part of a translation table walk for a stage 1 translation,
  349. * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
  350. * is UNKNOWN.
  351. */
  352. /* Check for permission fault, and S1PTW */
  353. mrc p15, 4, r1, c5, c2, 0 @ HSR
  354. and r0, r1, #HSR_FSC_TYPE
  355. cmp r0, #FSC_PERM
  356. tsteq r1, #(1 << 7) @ S1PTW
  357. mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
  358. bne 3f
  359. /* Preserve PAR */
  360. mrrc p15, 0, r0, r1, c7 @ PAR
  361. push {r0, r1}
  362. /* Resolve IPA using the xFAR */
  363. mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
  364. isb
  365. mrrc p15, 0, r0, r1, c7 @ PAR
  366. tst r0, #1
  367. bne 4f @ Failed translation
  368. ubfx r2, r0, #12, #20
  369. lsl r2, r2, #4
  370. orr r2, r2, r1, lsl #24
  371. /* Restore PAR */
  372. pop {r0, r1}
  373. mcrr p15, 0, r0, r1, c7 @ PAR
  374. 3: load_vcpu @ Load VCPU pointer to r0
  375. str r2, [r0, #VCPU_HPFAR]
  376. 1: mov r1, #ARM_EXCEPTION_HVC
  377. b __kvm_vcpu_return
  378. 4: pop {r0, r1} @ Failed translation, return to guest
  379. mcrr p15, 0, r0, r1, c7 @ PAR
  380. clrex
  381. pop {r0, r1, r2}
  382. eret
  383. /*
  384. * If VFPv3 support is not available, then we will not switch the VFP
  385. * registers; however cp10 and cp11 accesses will still trap and fallback
  386. * to the regular coprocessor emulation code, which currently will
  387. * inject an undefined exception to the guest.
  388. */
  389. #ifdef CONFIG_VFPv3
  390. switch_to_guest_vfp:
  391. load_vcpu @ Load VCPU pointer to r0
  392. push {r3-r7}
  393. @ NEON/VFP used. Turn on VFP access.
  394. set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
  395. @ Switch VFP/NEON hardware state to the guest's
  396. add r7, r0, #VCPU_VFP_HOST
  397. ldr r7, [r7]
  398. store_vfp_state r7
  399. add r7, r0, #VCPU_VFP_GUEST
  400. restore_vfp_state r7
  401. pop {r3-r7}
  402. pop {r0-r2}
  403. clrex
  404. eret
  405. #endif
  406. .align
  407. hyp_irq:
  408. push {r0, r1, r2}
  409. mov r1, #ARM_EXCEPTION_IRQ
  410. load_vcpu @ Load VCPU pointer to r0
  411. b __kvm_vcpu_return
  412. .align
  413. hyp_fiq:
  414. b hyp_fiq
  415. .ltorg
  416. __kvm_hyp_code_end:
  417. .globl __kvm_hyp_code_end
  418. .section ".rodata"
  419. und_die_str:
  420. .ascii "unexpected undefined exception in Hyp mode at: %#08x"
  421. pabt_die_str:
  422. .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
  423. dabt_die_str:
  424. .ascii "unexpected data abort in Hyp mode at: %#08x"
  425. svc_die_str:
  426. .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"