book3s_hv_rmhandlers.S 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881
  1. /*
  2. * This program is free software; you can redistribute it and/or modify
  3. * it under the terms of the GNU General Public License, version 2, as
  4. * published by the Free Software Foundation.
  5. *
  6. * This program is distributed in the hope that it will be useful,
  7. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  8. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  9. * GNU General Public License for more details.
  10. *
  11. * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  12. *
  13. * Derived from book3s_rmhandlers.S and other files, which are:
  14. *
  15. * Copyright SUSE Linux Products GmbH 2009
  16. *
  17. * Authors: Alexander Graf <agraf@suse.de>
  18. */
  19. #include <asm/ppc_asm.h>
  20. #include <asm/kvm_asm.h>
  21. #include <asm/reg.h>
  22. #include <asm/mmu.h>
  23. #include <asm/page.h>
  24. #include <asm/ptrace.h>
  25. #include <asm/hvcall.h>
  26. #include <asm/asm-offsets.h>
  27. #include <asm/exception-64s.h>
  28. #include <asm/kvm_book3s_asm.h>
  29. #include <asm/mmu-hash64.h>
  30. #ifdef __LITTLE_ENDIAN__
  31. #error Need to fix lppaca and SLB shadow accesses in little endian mode
  32. #endif
  33. /*****************************************************************************
  34. * *
  35. * Real Mode handlers that need to be in the linear mapping *
  36. * *
  37. ****************************************************************************/
  38. .globl kvmppc_skip_interrupt
  39. kvmppc_skip_interrupt:
  40. mfspr r13,SPRN_SRR0
  41. addi r13,r13,4
  42. mtspr SPRN_SRR0,r13
  43. GET_SCRATCH0(r13)
  44. rfid
  45. b .
  46. .globl kvmppc_skip_Hinterrupt
  47. kvmppc_skip_Hinterrupt:
  48. mfspr r13,SPRN_HSRR0
  49. addi r13,r13,4
  50. mtspr SPRN_HSRR0,r13
  51. GET_SCRATCH0(r13)
  52. hrfid
  53. b .
  54. /*
  55. * Call kvmppc_hv_entry in real mode.
  56. * Must be called with interrupts hard-disabled.
  57. *
  58. * Input Registers:
  59. *
  60. * LR = return address to continue at after eventually re-enabling MMU
  61. */
  62. _GLOBAL(kvmppc_hv_entry_trampoline)
  63. mfmsr r10
  64. LOAD_REG_ADDR(r5, kvmppc_hv_entry)
  65. li r0,MSR_RI
  66. andc r0,r10,r0
  67. li r6,MSR_IR | MSR_DR
  68. andc r6,r10,r6
  69. mtmsrd r0,1 /* clear RI in MSR */
  70. mtsrr0 r5
  71. mtsrr1 r6
  72. RFI
  73. /******************************************************************************
  74. * *
  75. * Entry code *
  76. * *
  77. *****************************************************************************/
  78. /*
  79. * We come in here when wakened from nap mode on a secondary hw thread.
  80. * Relocation is off and most register values are lost.
  81. * r13 points to the PACA.
  82. */
  83. .globl kvm_start_guest
  84. kvm_start_guest:
  85. ld r1,PACAEMERGSP(r13)
  86. subi r1,r1,STACK_FRAME_OVERHEAD
  87. ld r2,PACATOC(r13)
  88. li r0,KVM_HWTHREAD_IN_KVM
  89. stb r0,HSTATE_HWTHREAD_STATE(r13)
  90. /* NV GPR values from power7_idle() will no longer be valid */
  91. li r0,1
  92. stb r0,PACA_NAPSTATELOST(r13)
  93. /* were we napping due to cede? */
  94. lbz r0,HSTATE_NAPPING(r13)
  95. cmpwi r0,0
  96. bne kvm_end_cede
  97. /*
  98. * We weren't napping due to cede, so this must be a secondary
  99. * thread being woken up to run a guest, or being woken up due
  100. * to a stray IPI. (Or due to some machine check or hypervisor
  101. * maintenance interrupt while the core is in KVM.)
  102. */
  103. /* Check the wake reason in SRR1 to see why we got here */
  104. mfspr r3,SPRN_SRR1
  105. rlwinm r3,r3,44-31,0x7 /* extract wake reason field */
  106. cmpwi r3,4 /* was it an external interrupt? */
  107. bne 27f /* if not */
  108. ld r5,HSTATE_XICS_PHYS(r13)
  109. li r7,XICS_XIRR /* if it was an external interrupt, */
  110. lwzcix r8,r5,r7 /* get and ack the interrupt */
  111. sync
  112. clrldi. r9,r8,40 /* get interrupt source ID. */
  113. beq 28f /* none there? */
  114. cmpwi r9,XICS_IPI /* was it an IPI? */
  115. bne 29f
  116. li r0,0xff
  117. li r6,XICS_MFRR
  118. stbcix r0,r5,r6 /* clear IPI */
  119. stwcix r8,r5,r7 /* EOI the interrupt */
  120. sync /* order loading of vcpu after that */
  121. /* get vcpu pointer, NULL if we have no vcpu to run */
  122. ld r4,HSTATE_KVM_VCPU(r13)
  123. cmpdi r4,0
  124. /* if we have no vcpu to run, go back to sleep */
  125. beq kvm_no_guest
  126. b kvmppc_hv_entry
  127. 27: /* XXX should handle hypervisor maintenance interrupts etc. here */
  128. b kvm_no_guest
  129. 28: /* SRR1 said external but ICP said nope?? */
  130. b kvm_no_guest
  131. 29: /* External non-IPI interrupt to offline secondary thread? help?? */
  132. stw r8,HSTATE_SAVED_XIRR(r13)
  133. b kvm_no_guest
  134. .global kvmppc_hv_entry
  135. kvmppc_hv_entry:
  136. /* Required state:
  137. *
  138. * R4 = vcpu pointer
  139. * MSR = ~IR|DR
  140. * R13 = PACA
  141. * R1 = host R1
  142. * all other volatile GPRS = free
  143. */
  144. mflr r0
  145. std r0, HSTATE_VMHANDLER(r13)
  146. /* Set partition DABR */
  147. /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
  148. li r5,3
  149. ld r6,VCPU_DABR(r4)
  150. mtspr SPRN_DABRX,r5
  151. mtspr SPRN_DABR,r6
  152. BEGIN_FTR_SECTION
  153. isync
  154. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  155. /* Load guest PMU registers */
  156. /* R4 is live here (vcpu pointer) */
  157. li r3, 1
  158. sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
  159. mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
  160. isync
  161. lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
  162. lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
  163. lwz r6, VCPU_PMC + 8(r4)
  164. lwz r7, VCPU_PMC + 12(r4)
  165. lwz r8, VCPU_PMC + 16(r4)
  166. lwz r9, VCPU_PMC + 20(r4)
  167. BEGIN_FTR_SECTION
  168. lwz r10, VCPU_PMC + 24(r4)
  169. lwz r11, VCPU_PMC + 28(r4)
  170. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  171. mtspr SPRN_PMC1, r3
  172. mtspr SPRN_PMC2, r5
  173. mtspr SPRN_PMC3, r6
  174. mtspr SPRN_PMC4, r7
  175. mtspr SPRN_PMC5, r8
  176. mtspr SPRN_PMC6, r9
  177. BEGIN_FTR_SECTION
  178. mtspr SPRN_PMC7, r10
  179. mtspr SPRN_PMC8, r11
  180. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  181. ld r3, VCPU_MMCR(r4)
  182. ld r5, VCPU_MMCR + 8(r4)
  183. ld r6, VCPU_MMCR + 16(r4)
  184. mtspr SPRN_MMCR1, r5
  185. mtspr SPRN_MMCRA, r6
  186. mtspr SPRN_MMCR0, r3
  187. isync
  188. /* Load up FP, VMX and VSX registers */
  189. bl kvmppc_load_fp
  190. ld r14, VCPU_GPR(R14)(r4)
  191. ld r15, VCPU_GPR(R15)(r4)
  192. ld r16, VCPU_GPR(R16)(r4)
  193. ld r17, VCPU_GPR(R17)(r4)
  194. ld r18, VCPU_GPR(R18)(r4)
  195. ld r19, VCPU_GPR(R19)(r4)
  196. ld r20, VCPU_GPR(R20)(r4)
  197. ld r21, VCPU_GPR(R21)(r4)
  198. ld r22, VCPU_GPR(R22)(r4)
  199. ld r23, VCPU_GPR(R23)(r4)
  200. ld r24, VCPU_GPR(R24)(r4)
  201. ld r25, VCPU_GPR(R25)(r4)
  202. ld r26, VCPU_GPR(R26)(r4)
  203. ld r27, VCPU_GPR(R27)(r4)
  204. ld r28, VCPU_GPR(R28)(r4)
  205. ld r29, VCPU_GPR(R29)(r4)
  206. ld r30, VCPU_GPR(R30)(r4)
  207. ld r31, VCPU_GPR(R31)(r4)
  208. BEGIN_FTR_SECTION
  209. /* Switch DSCR to guest value */
  210. ld r5, VCPU_DSCR(r4)
  211. mtspr SPRN_DSCR, r5
  212. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  213. /*
  214. * Set the decrementer to the guest decrementer.
  215. */
  216. ld r8,VCPU_DEC_EXPIRES(r4)
  217. mftb r7
  218. subf r3,r7,r8
  219. mtspr SPRN_DEC,r3
  220. stw r3,VCPU_DEC(r4)
  221. ld r5, VCPU_SPRG0(r4)
  222. ld r6, VCPU_SPRG1(r4)
  223. ld r7, VCPU_SPRG2(r4)
  224. ld r8, VCPU_SPRG3(r4)
  225. mtspr SPRN_SPRG0, r5
  226. mtspr SPRN_SPRG1, r6
  227. mtspr SPRN_SPRG2, r7
  228. mtspr SPRN_SPRG3, r8
  229. /* Save R1 in the PACA */
  230. std r1, HSTATE_HOST_R1(r13)
  231. /* Increment yield count if they have a VPA */
  232. ld r3, VCPU_VPA(r4)
  233. cmpdi r3, 0
  234. beq 25f
  235. lwz r5, LPPACA_YIELDCOUNT(r3)
  236. addi r5, r5, 1
  237. stw r5, LPPACA_YIELDCOUNT(r3)
  238. li r6, 1
  239. stb r6, VCPU_VPA_DIRTY(r4)
  240. 25:
  241. /* Load up DAR and DSISR */
  242. ld r5, VCPU_DAR(r4)
  243. lwz r6, VCPU_DSISR(r4)
  244. mtspr SPRN_DAR, r5
  245. mtspr SPRN_DSISR, r6
  246. BEGIN_FTR_SECTION
  247. /* Restore AMR and UAMOR, set AMOR to all 1s */
  248. ld r5,VCPU_AMR(r4)
  249. ld r6,VCPU_UAMOR(r4)
  250. li r7,-1
  251. mtspr SPRN_AMR,r5
  252. mtspr SPRN_UAMOR,r6
  253. mtspr SPRN_AMOR,r7
  254. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  255. /* Clear out SLB */
  256. li r6,0
  257. slbmte r6,r6
  258. slbia
  259. ptesync
  260. BEGIN_FTR_SECTION
  261. b 30f
  262. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  263. /*
  264. * POWER7 host -> guest partition switch code.
  265. * We don't have to lock against concurrent tlbies,
  266. * but we do have to coordinate across hardware threads.
  267. */
  268. /* Increment entry count iff exit count is zero. */
  269. ld r5,HSTATE_KVM_VCORE(r13)
  270. addi r9,r5,VCORE_ENTRY_EXIT
  271. 21: lwarx r3,0,r9
  272. cmpwi r3,0x100 /* any threads starting to exit? */
  273. bge secondary_too_late /* if so we're too late to the party */
  274. addi r3,r3,1
  275. stwcx. r3,0,r9
  276. bne 21b
  277. /* Primary thread switches to guest partition. */
  278. ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
  279. lwz r6,VCPU_PTID(r4)
  280. cmpwi r6,0
  281. bne 20f
  282. ld r6,KVM_SDR1(r9)
  283. lwz r7,KVM_LPID(r9)
  284. li r0,LPID_RSVD /* switch to reserved LPID */
  285. mtspr SPRN_LPID,r0
  286. ptesync
  287. mtspr SPRN_SDR1,r6 /* switch to partition page table */
  288. mtspr SPRN_LPID,r7
  289. isync
  290. /* See if we need to flush the TLB */
  291. lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
  292. clrldi r7,r6,64-6 /* extract bit number (6 bits) */
  293. srdi r6,r6,6 /* doubleword number */
  294. sldi r6,r6,3 /* address offset */
  295. add r6,r6,r9
  296. addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
  297. li r0,1
  298. sld r0,r0,r7
  299. ld r7,0(r6)
  300. and. r7,r7,r0
  301. beq 22f
  302. 23: ldarx r7,0,r6 /* if set, clear the bit */
  303. andc r7,r7,r0
  304. stdcx. r7,0,r6
  305. bne 23b
  306. li r6,128 /* and flush the TLB */
  307. mtctr r6
  308. li r7,0x800 /* IS field = 0b10 */
  309. ptesync
  310. 28: tlbiel r7
  311. addi r7,r7,0x1000
  312. bdnz 28b
  313. ptesync
  314. 22: li r0,1
  315. stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
  316. b 10f
  317. /* Secondary threads wait for primary to have done partition switch */
  318. 20: lbz r0,VCORE_IN_GUEST(r5)
  319. cmpwi r0,0
  320. beq 20b
  321. /* Set LPCR and RMOR. */
  322. 10: ld r8,KVM_LPCR(r9)
  323. mtspr SPRN_LPCR,r8
  324. ld r8,KVM_RMOR(r9)
  325. mtspr SPRN_RMOR,r8
  326. isync
  327. /* Check if HDEC expires soon */
  328. mfspr r3,SPRN_HDEC
  329. cmpwi r3,10
  330. li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
  331. mr r9,r4
  332. blt hdec_soon
  333. /* Save purr/spurr */
  334. mfspr r5,SPRN_PURR
  335. mfspr r6,SPRN_SPURR
  336. std r5,HSTATE_PURR(r13)
  337. std r6,HSTATE_SPURR(r13)
  338. ld r7,VCPU_PURR(r4)
  339. ld r8,VCPU_SPURR(r4)
  340. mtspr SPRN_PURR,r7
  341. mtspr SPRN_SPURR,r8
  342. b 31f
  343. /*
  344. * PPC970 host -> guest partition switch code.
  345. * We have to lock against concurrent tlbies,
  346. * using native_tlbie_lock to lock against host tlbies
  347. * and kvm->arch.tlbie_lock to lock against guest tlbies.
  348. * We also have to invalidate the TLB since its
  349. * entries aren't tagged with the LPID.
  350. */
  351. 30: ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
  352. /* first take native_tlbie_lock */
  353. .section ".toc","aw"
  354. toc_tlbie_lock:
  355. .tc native_tlbie_lock[TC],native_tlbie_lock
  356. .previous
  357. ld r3,toc_tlbie_lock@toc(2)
  358. #ifdef __BIG_ENDIAN__
  359. lwz r8,PACA_LOCK_TOKEN(r13)
  360. #else
  361. lwz r8,PACAPACAINDEX(r13)
  362. #endif
  363. 24: lwarx r0,0,r3
  364. cmpwi r0,0
  365. bne 24b
  366. stwcx. r8,0,r3
  367. bne 24b
  368. isync
  369. ld r7,KVM_LPCR(r9) /* use kvm->arch.lpcr to store HID4 */
  370. li r0,0x18f
  371. rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
  372. or r0,r7,r0
  373. ptesync
  374. sync
  375. mtspr SPRN_HID4,r0 /* switch to reserved LPID */
  376. isync
  377. li r0,0
  378. stw r0,0(r3) /* drop native_tlbie_lock */
  379. /* invalidate the whole TLB */
  380. li r0,256
  381. mtctr r0
  382. li r6,0
  383. 25: tlbiel r6
  384. addi r6,r6,0x1000
  385. bdnz 25b
  386. ptesync
  387. /* Take the guest's tlbie_lock */
  388. addi r3,r9,KVM_TLBIE_LOCK
  389. 24: lwarx r0,0,r3
  390. cmpwi r0,0
  391. bne 24b
  392. stwcx. r8,0,r3
  393. bne 24b
  394. isync
  395. ld r6,KVM_SDR1(r9)
  396. mtspr SPRN_SDR1,r6 /* switch to partition page table */
  397. /* Set up HID4 with the guest's LPID etc. */
  398. sync
  399. mtspr SPRN_HID4,r7
  400. isync
  401. /* drop the guest's tlbie_lock */
  402. li r0,0
  403. stw r0,0(r3)
  404. /* Check if HDEC expires soon */
  405. mfspr r3,SPRN_HDEC
  406. cmpwi r3,10
  407. li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
  408. mr r9,r4
  409. blt hdec_soon
  410. /* Enable HDEC interrupts */
  411. mfspr r0,SPRN_HID0
  412. li r3,1
  413. rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
  414. sync
  415. mtspr SPRN_HID0,r0
  416. mfspr r0,SPRN_HID0
  417. mfspr r0,SPRN_HID0
  418. mfspr r0,SPRN_HID0
  419. mfspr r0,SPRN_HID0
  420. mfspr r0,SPRN_HID0
  421. mfspr r0,SPRN_HID0
  422. /* Load up guest SLB entries */
  423. 31: lwz r5,VCPU_SLB_MAX(r4)
  424. cmpwi r5,0
  425. beq 9f
  426. mtctr r5
  427. addi r6,r4,VCPU_SLB
  428. 1: ld r8,VCPU_SLB_E(r6)
  429. ld r9,VCPU_SLB_V(r6)
  430. slbmte r9,r8
  431. addi r6,r6,VCPU_SLB_SIZE
  432. bdnz 1b
  433. 9:
  434. /* Restore state of CTRL run bit; assume 1 on entry */
  435. lwz r5,VCPU_CTRL(r4)
  436. andi. r5,r5,1
  437. bne 4f
  438. mfspr r6,SPRN_CTRLF
  439. clrrdi r6,r6,1
  440. mtspr SPRN_CTRLT,r6
  441. 4:
  442. ld r6, VCPU_CTR(r4)
  443. lwz r7, VCPU_XER(r4)
  444. mtctr r6
  445. mtxer r7
  446. ld r10, VCPU_PC(r4)
  447. ld r11, VCPU_MSR(r4)
  448. kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
  449. ld r6, VCPU_SRR0(r4)
  450. ld r7, VCPU_SRR1(r4)
  451. /* r11 = vcpu->arch.msr & ~MSR_HV */
  452. rldicl r11, r11, 63 - MSR_HV_LG, 1
  453. rotldi r11, r11, 1 + MSR_HV_LG
  454. ori r11, r11, MSR_ME
  455. /* Check if we can deliver an external or decrementer interrupt now */
  456. ld r0,VCPU_PENDING_EXC(r4)
  457. lis r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
  458. and r0,r0,r8
  459. cmpdi cr1,r0,0
  460. andi. r0,r11,MSR_EE
  461. beq cr1,11f
  462. BEGIN_FTR_SECTION
  463. mfspr r8,SPRN_LPCR
  464. ori r8,r8,LPCR_MER
  465. mtspr SPRN_LPCR,r8
  466. isync
  467. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  468. beq 5f
  469. li r0,BOOK3S_INTERRUPT_EXTERNAL
  470. 12: mr r6,r10
  471. mr r10,r0
  472. mr r7,r11
  473. li r11,(MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
  474. rotldi r11,r11,63
  475. b 5f
  476. 11: beq 5f
  477. mfspr r0,SPRN_DEC
  478. cmpwi r0,0
  479. li r0,BOOK3S_INTERRUPT_DECREMENTER
  480. blt 12b
  481. /* Move SRR0 and SRR1 into the respective regs */
  482. 5: mtspr SPRN_SRR0, r6
  483. mtspr SPRN_SRR1, r7
  484. fast_guest_return:
  485. li r0,0
  486. stb r0,VCPU_CEDED(r4) /* cancel cede */
  487. mtspr SPRN_HSRR0,r10
  488. mtspr SPRN_HSRR1,r11
  489. /* Activate guest mode, so faults get handled by KVM */
  490. li r9, KVM_GUEST_MODE_GUEST
  491. stb r9, HSTATE_IN_GUEST(r13)
  492. /* Enter guest */
  493. BEGIN_FTR_SECTION
  494. ld r5, VCPU_CFAR(r4)
  495. mtspr SPRN_CFAR, r5
  496. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  497. ld r5, VCPU_LR(r4)
  498. lwz r6, VCPU_CR(r4)
  499. mtlr r5
  500. mtcr r6
  501. ld r0, VCPU_GPR(R0)(r4)
  502. ld r1, VCPU_GPR(R1)(r4)
  503. ld r2, VCPU_GPR(R2)(r4)
  504. ld r3, VCPU_GPR(R3)(r4)
  505. ld r5, VCPU_GPR(R5)(r4)
  506. ld r6, VCPU_GPR(R6)(r4)
  507. ld r7, VCPU_GPR(R7)(r4)
  508. ld r8, VCPU_GPR(R8)(r4)
  509. ld r9, VCPU_GPR(R9)(r4)
  510. ld r10, VCPU_GPR(R10)(r4)
  511. ld r11, VCPU_GPR(R11)(r4)
  512. ld r12, VCPU_GPR(R12)(r4)
  513. ld r13, VCPU_GPR(R13)(r4)
  514. ld r4, VCPU_GPR(R4)(r4)
  515. hrfid
  516. b .
  517. /******************************************************************************
  518. * *
  519. * Exit code *
  520. * *
  521. *****************************************************************************/
  522. /*
  523. * We come here from the first-level interrupt handlers.
  524. */
  525. .globl kvmppc_interrupt
  526. kvmppc_interrupt:
  527. /*
  528. * Register contents:
  529. * R12 = interrupt vector
  530. * R13 = PACA
  531. * guest CR, R12 saved in shadow VCPU SCRATCH1/0
  532. * guest R13 saved in SPRN_SCRATCH0
  533. */
  534. /* abuse host_r2 as third scratch area; we get r2 from PACATOC(r13) */
  535. std r9, HSTATE_HOST_R2(r13)
  536. ld r9, HSTATE_KVM_VCPU(r13)
  537. /* Save registers */
  538. std r0, VCPU_GPR(R0)(r9)
  539. std r1, VCPU_GPR(R1)(r9)
  540. std r2, VCPU_GPR(R2)(r9)
  541. std r3, VCPU_GPR(R3)(r9)
  542. std r4, VCPU_GPR(R4)(r9)
  543. std r5, VCPU_GPR(R5)(r9)
  544. std r6, VCPU_GPR(R6)(r9)
  545. std r7, VCPU_GPR(R7)(r9)
  546. std r8, VCPU_GPR(R8)(r9)
  547. ld r0, HSTATE_HOST_R2(r13)
  548. std r0, VCPU_GPR(R9)(r9)
  549. std r10, VCPU_GPR(R10)(r9)
  550. std r11, VCPU_GPR(R11)(r9)
  551. ld r3, HSTATE_SCRATCH0(r13)
  552. lwz r4, HSTATE_SCRATCH1(r13)
  553. std r3, VCPU_GPR(R12)(r9)
  554. stw r4, VCPU_CR(r9)
  555. BEGIN_FTR_SECTION
  556. ld r3, HSTATE_CFAR(r13)
  557. std r3, VCPU_CFAR(r9)
  558. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  559. /* Restore R1/R2 so we can handle faults */
  560. ld r1, HSTATE_HOST_R1(r13)
  561. ld r2, PACATOC(r13)
  562. mfspr r10, SPRN_SRR0
  563. mfspr r11, SPRN_SRR1
  564. std r10, VCPU_SRR0(r9)
  565. std r11, VCPU_SRR1(r9)
  566. andi. r0, r12, 2 /* need to read HSRR0/1? */
  567. beq 1f
  568. mfspr r10, SPRN_HSRR0
  569. mfspr r11, SPRN_HSRR1
  570. clrrdi r12, r12, 2
  571. 1: std r10, VCPU_PC(r9)
  572. std r11, VCPU_MSR(r9)
  573. GET_SCRATCH0(r3)
  574. mflr r4
  575. std r3, VCPU_GPR(R13)(r9)
  576. std r4, VCPU_LR(r9)
  577. /* Unset guest mode */
  578. li r0, KVM_GUEST_MODE_NONE
  579. stb r0, HSTATE_IN_GUEST(r13)
  580. stw r12,VCPU_TRAP(r9)
  581. /* Save HEIR (HV emulation assist reg) in last_inst
  582. if this is an HEI (HV emulation interrupt, e40) */
  583. li r3,KVM_INST_FETCH_FAILED
  584. BEGIN_FTR_SECTION
  585. cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
  586. bne 11f
  587. mfspr r3,SPRN_HEIR
  588. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  589. 11: stw r3,VCPU_LAST_INST(r9)
  590. /* these are volatile across C function calls */
  591. mfctr r3
  592. mfxer r4
  593. std r3, VCPU_CTR(r9)
  594. stw r4, VCPU_XER(r9)
  595. BEGIN_FTR_SECTION
  596. /* If this is a page table miss then see if it's theirs or ours */
  597. cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
  598. beq kvmppc_hdsi
  599. cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
  600. beq kvmppc_hisi
  601. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  602. /* See if this is a leftover HDEC interrupt */
  603. cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
  604. bne 2f
  605. mfspr r3,SPRN_HDEC
  606. cmpwi r3,0
  607. bge ignore_hdec
  608. 2:
  609. /* See if this is an hcall we can handle in real mode */
  610. cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
  611. beq hcall_try_real_mode
  612. /* Only handle external interrupts here on arch 206 and later */
  613. BEGIN_FTR_SECTION
  614. b ext_interrupt_to_host
  615. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
  616. /* External interrupt ? */
  617. cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
  618. bne+ ext_interrupt_to_host
  619. /* External interrupt, first check for host_ipi. If this is
  620. * set, we know the host wants us out so let's do it now
  621. */
  622. do_ext_interrupt:
  623. lbz r0, HSTATE_HOST_IPI(r13)
  624. cmpwi r0, 0
  625. bne ext_interrupt_to_host
  626. /* Now read the interrupt from the ICP */
  627. ld r5, HSTATE_XICS_PHYS(r13)
  628. li r7, XICS_XIRR
  629. cmpdi r5, 0
  630. beq- ext_interrupt_to_host
  631. lwzcix r3, r5, r7
  632. rlwinm. r0, r3, 0, 0xffffff
  633. sync
  634. beq 3f /* if nothing pending in the ICP */
  635. /* We found something in the ICP...
  636. *
  637. * If it's not an IPI, stash it in the PACA and return to
  638. * the host, we don't (yet) handle directing real external
  639. * interrupts directly to the guest
  640. */
  641. cmpwi r0, XICS_IPI
  642. bne ext_stash_for_host
  643. /* It's an IPI, clear the MFRR and EOI it */
  644. li r0, 0xff
  645. li r6, XICS_MFRR
  646. stbcix r0, r5, r6 /* clear the IPI */
  647. stwcix r3, r5, r7 /* EOI it */
  648. sync
  649. /* We need to re-check host IPI now in case it got set in the
  650. * meantime. If it's clear, we bounce the interrupt to the
  651. * guest
  652. */
  653. lbz r0, HSTATE_HOST_IPI(r13)
  654. cmpwi r0, 0
  655. bne- 1f
  656. /* Allright, looks like an IPI for the guest, we need to set MER */
  657. 3:
  658. /* Check if any CPU is heading out to the host, if so head out too */
  659. ld r5, HSTATE_KVM_VCORE(r13)
  660. lwz r0, VCORE_ENTRY_EXIT(r5)
  661. cmpwi r0, 0x100
  662. bge ext_interrupt_to_host
  663. /* See if there is a pending interrupt for the guest */
  664. mfspr r8, SPRN_LPCR
  665. ld r0, VCPU_PENDING_EXC(r9)
  666. /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
  667. rldicl. r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
  668. rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
  669. beq 2f
  670. /* And if the guest EE is set, we can deliver immediately, else
  671. * we return to the guest with MER set
  672. */
  673. andi. r0, r11, MSR_EE
  674. beq 2f
  675. mtspr SPRN_SRR0, r10
  676. mtspr SPRN_SRR1, r11
  677. li r10, BOOK3S_INTERRUPT_EXTERNAL
  678. li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
  679. rotldi r11, r11, 63
  680. 2: mr r4, r9
  681. mtspr SPRN_LPCR, r8
  682. b fast_guest_return
  683. /* We raced with the host, we need to resend that IPI, bummer */
  684. 1: li r0, IPI_PRIORITY
  685. stbcix r0, r5, r6 /* set the IPI */
  686. sync
  687. b ext_interrupt_to_host
  688. ext_stash_for_host:
  689. /* It's not an IPI and it's for the host, stash it in the PACA
  690. * before exit, it will be picked up by the host ICP driver
  691. */
  692. stw r3, HSTATE_SAVED_XIRR(r13)
  693. ext_interrupt_to_host:
  694. guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
  695. /* Save DEC */
  696. mfspr r5,SPRN_DEC
  697. mftb r6
  698. extsw r5,r5
  699. add r5,r5,r6
  700. std r5,VCPU_DEC_EXPIRES(r9)
  701. /* Save more register state */
  702. mfdar r6
  703. mfdsisr r7
  704. std r6, VCPU_DAR(r9)
  705. stw r7, VCPU_DSISR(r9)
  706. BEGIN_FTR_SECTION
  707. /* don't overwrite fault_dar/fault_dsisr if HDSI */
  708. cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
  709. beq 6f
  710. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  711. std r6, VCPU_FAULT_DAR(r9)
  712. stw r7, VCPU_FAULT_DSISR(r9)
  713. /* See if it is a machine check */
  714. cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
  715. beq machine_check_realmode
  716. mc_cont:
  717. /* Save guest CTRL register, set runlatch to 1 */
  718. 6: mfspr r6,SPRN_CTRLF
  719. stw r6,VCPU_CTRL(r9)
  720. andi. r0,r6,1
  721. bne 4f
  722. ori r6,r6,1
  723. mtspr SPRN_CTRLT,r6
  724. 4:
  725. /* Read the guest SLB and save it away */
  726. lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
  727. mtctr r0
  728. li r6,0
  729. addi r7,r9,VCPU_SLB
  730. li r5,0
  731. 1: slbmfee r8,r6
  732. andis. r0,r8,SLB_ESID_V@h
  733. beq 2f
  734. add r8,r8,r6 /* put index in */
  735. slbmfev r3,r6
  736. std r8,VCPU_SLB_E(r7)
  737. std r3,VCPU_SLB_V(r7)
  738. addi r7,r7,VCPU_SLB_SIZE
  739. addi r5,r5,1
  740. 2: addi r6,r6,1
  741. bdnz 1b
  742. stw r5,VCPU_SLB_MAX(r9)
  743. /*
  744. * Save the guest PURR/SPURR
  745. */
  746. BEGIN_FTR_SECTION
  747. mfspr r5,SPRN_PURR
  748. mfspr r6,SPRN_SPURR
  749. ld r7,VCPU_PURR(r9)
  750. ld r8,VCPU_SPURR(r9)
  751. std r5,VCPU_PURR(r9)
  752. std r6,VCPU_SPURR(r9)
  753. subf r5,r7,r5
  754. subf r6,r8,r6
  755. /*
  756. * Restore host PURR/SPURR and add guest times
  757. * so that the time in the guest gets accounted.
  758. */
  759. ld r3,HSTATE_PURR(r13)
  760. ld r4,HSTATE_SPURR(r13)
  761. add r3,r3,r5
  762. add r4,r4,r6
  763. mtspr SPRN_PURR,r3
  764. mtspr SPRN_SPURR,r4
  765. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
  766. /* Clear out SLB */
  767. li r5,0
  768. slbmte r5,r5
  769. slbia
  770. ptesync
  771. hdec_soon: /* r9 = vcpu, r12 = trap, r13 = paca */
  772. BEGIN_FTR_SECTION
  773. b 32f
  774. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  775. /*
  776. * POWER7 guest -> host partition switch code.
  777. * We don't have to lock against tlbies but we do
  778. * have to coordinate the hardware threads.
  779. */
  780. /* Increment the threads-exiting-guest count in the 0xff00
  781. bits of vcore->entry_exit_count */
  782. lwsync
  783. ld r5,HSTATE_KVM_VCORE(r13)
  784. addi r6,r5,VCORE_ENTRY_EXIT
  785. 41: lwarx r3,0,r6
  786. addi r0,r3,0x100
  787. stwcx. r0,0,r6
  788. bne 41b
  789. lwsync
  790. /*
  791. * At this point we have an interrupt that we have to pass
  792. * up to the kernel or qemu; we can't handle it in real mode.
  793. * Thus we have to do a partition switch, so we have to
  794. * collect the other threads, if we are the first thread
  795. * to take an interrupt. To do this, we set the HDEC to 0,
  796. * which causes an HDEC interrupt in all threads within 2ns
  797. * because the HDEC register is shared between all 4 threads.
  798. * However, we don't need to bother if this is an HDEC
  799. * interrupt, since the other threads will already be on their
  800. * way here in that case.
  801. */
  802. cmpwi r3,0x100 /* Are we the first here? */
  803. bge 43f
  804. cmpwi r3,1 /* Are any other threads in the guest? */
  805. ble 43f
  806. cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
  807. beq 40f
  808. li r0,0
  809. mtspr SPRN_HDEC,r0
  810. 40:
  811. /*
  812. * Send an IPI to any napping threads, since an HDEC interrupt
  813. * doesn't wake CPUs up from nap.
  814. */
  815. lwz r3,VCORE_NAPPING_THREADS(r5)
  816. lwz r4,VCPU_PTID(r9)
  817. li r0,1
  818. sld r0,r0,r4
  819. andc. r3,r3,r0 /* no sense IPI'ing ourselves */
  820. beq 43f
  821. mulli r4,r4,PACA_SIZE /* get paca for thread 0 */
  822. subf r6,r4,r13
  823. 42: andi. r0,r3,1
  824. beq 44f
  825. ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */
  826. li r0,IPI_PRIORITY
  827. li r7,XICS_MFRR
  828. stbcix r0,r7,r8 /* trigger the IPI */
  829. 44: srdi. r3,r3,1
  830. addi r6,r6,PACA_SIZE
  831. bne 42b
  832. /* Secondary threads wait for primary to do partition switch */
  833. 43: ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
  834. ld r5,HSTATE_KVM_VCORE(r13)
  835. lwz r3,VCPU_PTID(r9)
  836. cmpwi r3,0
  837. beq 15f
  838. HMT_LOW
  839. 13: lbz r3,VCORE_IN_GUEST(r5)
  840. cmpwi r3,0
  841. bne 13b
  842. HMT_MEDIUM
  843. b 16f
  844. /* Primary thread waits for all the secondaries to exit guest */
  845. 15: lwz r3,VCORE_ENTRY_EXIT(r5)
  846. srwi r0,r3,8
  847. clrldi r3,r3,56
  848. cmpw r3,r0
  849. bne 15b
  850. isync
  851. /* Primary thread switches back to host partition */
  852. ld r6,KVM_HOST_SDR1(r4)
  853. lwz r7,KVM_HOST_LPID(r4)
  854. li r8,LPID_RSVD /* switch to reserved LPID */
  855. mtspr SPRN_LPID,r8
  856. ptesync
  857. mtspr SPRN_SDR1,r6 /* switch to partition page table */
  858. mtspr SPRN_LPID,r7
  859. isync
  860. li r0,0
  861. stb r0,VCORE_IN_GUEST(r5)
  862. lis r8,0x7fff /* MAX_INT@h */
  863. mtspr SPRN_HDEC,r8
  864. 16: ld r8,KVM_HOST_LPCR(r4)
  865. mtspr SPRN_LPCR,r8
  866. isync
  867. b 33f
  868. /*
  869. * PPC970 guest -> host partition switch code.
  870. * We have to lock against concurrent tlbies, and
  871. * we have to flush the whole TLB.
  872. */
  873. 32: ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
  874. /* Take the guest's tlbie_lock */
  875. #ifdef __BIG_ENDIAN__
  876. lwz r8,PACA_LOCK_TOKEN(r13)
  877. #else
  878. lwz r8,PACAPACAINDEX(r13)
  879. #endif
  880. addi r3,r4,KVM_TLBIE_LOCK
  881. 24: lwarx r0,0,r3
  882. cmpwi r0,0
  883. bne 24b
  884. stwcx. r8,0,r3
  885. bne 24b
  886. isync
  887. ld r7,KVM_HOST_LPCR(r4) /* use kvm->arch.host_lpcr for HID4 */
  888. li r0,0x18f
  889. rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
  890. or r0,r7,r0
  891. ptesync
  892. sync
  893. mtspr SPRN_HID4,r0 /* switch to reserved LPID */
  894. isync
  895. li r0,0
  896. stw r0,0(r3) /* drop guest tlbie_lock */
  897. /* invalidate the whole TLB */
  898. li r0,256
  899. mtctr r0
  900. li r6,0
  901. 25: tlbiel r6
  902. addi r6,r6,0x1000
  903. bdnz 25b
  904. ptesync
  905. /* take native_tlbie_lock */
  906. ld r3,toc_tlbie_lock@toc(2)
  907. 24: lwarx r0,0,r3
  908. cmpwi r0,0
  909. bne 24b
  910. stwcx. r8,0,r3
  911. bne 24b
  912. isync
  913. ld r6,KVM_HOST_SDR1(r4)
  914. mtspr SPRN_SDR1,r6 /* switch to host page table */
  915. /* Set up host HID4 value */
  916. sync
  917. mtspr SPRN_HID4,r7
  918. isync
  919. li r0,0
  920. stw r0,0(r3) /* drop native_tlbie_lock */
  921. lis r8,0x7fff /* MAX_INT@h */
  922. mtspr SPRN_HDEC,r8
  923. /* Disable HDEC interrupts */
  924. mfspr r0,SPRN_HID0
  925. li r3,0
  926. rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
  927. sync
  928. mtspr SPRN_HID0,r0
  929. mfspr r0,SPRN_HID0
  930. mfspr r0,SPRN_HID0
  931. mfspr r0,SPRN_HID0
  932. mfspr r0,SPRN_HID0
  933. mfspr r0,SPRN_HID0
  934. mfspr r0,SPRN_HID0
  935. /* load host SLB entries */
  936. 33: ld r8,PACA_SLBSHADOWPTR(r13)
  937. .rept SLB_NUM_BOLTED
  938. ld r5,SLBSHADOW_SAVEAREA(r8)
  939. ld r6,SLBSHADOW_SAVEAREA+8(r8)
  940. andis. r7,r5,SLB_ESID_V@h
  941. beq 1f
  942. slbmte r6,r5
  943. 1: addi r8,r8,16
  944. .endr
  945. /* Save and reset AMR and UAMOR before turning on the MMU */
  946. BEGIN_FTR_SECTION
  947. mfspr r5,SPRN_AMR
  948. mfspr r6,SPRN_UAMOR
  949. std r5,VCPU_AMR(r9)
  950. std r6,VCPU_UAMOR(r9)
  951. li r6,0
  952. mtspr SPRN_AMR,r6
  953. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  954. /* Switch DSCR back to host value */
  955. BEGIN_FTR_SECTION
  956. mfspr r8, SPRN_DSCR
  957. ld r7, HSTATE_DSCR(r13)
  958. std r8, VCPU_DSCR(r7)
  959. mtspr SPRN_DSCR, r7
  960. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  961. /* Save non-volatile GPRs */
  962. std r14, VCPU_GPR(R14)(r9)
  963. std r15, VCPU_GPR(R15)(r9)
  964. std r16, VCPU_GPR(R16)(r9)
  965. std r17, VCPU_GPR(R17)(r9)
  966. std r18, VCPU_GPR(R18)(r9)
  967. std r19, VCPU_GPR(R19)(r9)
  968. std r20, VCPU_GPR(R20)(r9)
  969. std r21, VCPU_GPR(R21)(r9)
  970. std r22, VCPU_GPR(R22)(r9)
  971. std r23, VCPU_GPR(R23)(r9)
  972. std r24, VCPU_GPR(R24)(r9)
  973. std r25, VCPU_GPR(R25)(r9)
  974. std r26, VCPU_GPR(R26)(r9)
  975. std r27, VCPU_GPR(R27)(r9)
  976. std r28, VCPU_GPR(R28)(r9)
  977. std r29, VCPU_GPR(R29)(r9)
  978. std r30, VCPU_GPR(R30)(r9)
  979. std r31, VCPU_GPR(R31)(r9)
  980. /* Save SPRGs */
  981. mfspr r3, SPRN_SPRG0
  982. mfspr r4, SPRN_SPRG1
  983. mfspr r5, SPRN_SPRG2
  984. mfspr r6, SPRN_SPRG3
  985. std r3, VCPU_SPRG0(r9)
  986. std r4, VCPU_SPRG1(r9)
  987. std r5, VCPU_SPRG2(r9)
  988. std r6, VCPU_SPRG3(r9)
  989. /* save FP state */
  990. mr r3, r9
  991. bl .kvmppc_save_fp
  992. /* Increment yield count if they have a VPA */
  993. ld r8, VCPU_VPA(r9) /* do they have a VPA? */
  994. cmpdi r8, 0
  995. beq 25f
  996. lwz r3, LPPACA_YIELDCOUNT(r8)
  997. addi r3, r3, 1
  998. stw r3, LPPACA_YIELDCOUNT(r8)
  999. li r3, 1
  1000. stb r3, VCPU_VPA_DIRTY(r9)
  1001. 25:
  1002. /* Save PMU registers if requested */
  1003. /* r8 and cr0.eq are live here */
  1004. li r3, 1
  1005. sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
  1006. mfspr r4, SPRN_MMCR0 /* save MMCR0 */
  1007. mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
  1008. mfspr r6, SPRN_MMCRA
  1009. BEGIN_FTR_SECTION
  1010. /* On P7, clear MMCRA in order to disable SDAR updates */
  1011. li r7, 0
  1012. mtspr SPRN_MMCRA, r7
  1013. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  1014. isync
  1015. beq 21f /* if no VPA, save PMU stuff anyway */
  1016. lbz r7, LPPACA_PMCINUSE(r8)
  1017. cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
  1018. bne 21f
  1019. std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
  1020. b 22f
  1021. 21: mfspr r5, SPRN_MMCR1
  1022. std r4, VCPU_MMCR(r9)
  1023. std r5, VCPU_MMCR + 8(r9)
  1024. std r6, VCPU_MMCR + 16(r9)
  1025. mfspr r3, SPRN_PMC1
  1026. mfspr r4, SPRN_PMC2
  1027. mfspr r5, SPRN_PMC3
  1028. mfspr r6, SPRN_PMC4
  1029. mfspr r7, SPRN_PMC5
  1030. mfspr r8, SPRN_PMC6
  1031. BEGIN_FTR_SECTION
  1032. mfspr r10, SPRN_PMC7
  1033. mfspr r11, SPRN_PMC8
  1034. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  1035. stw r3, VCPU_PMC(r9)
  1036. stw r4, VCPU_PMC + 4(r9)
  1037. stw r5, VCPU_PMC + 8(r9)
  1038. stw r6, VCPU_PMC + 12(r9)
  1039. stw r7, VCPU_PMC + 16(r9)
  1040. stw r8, VCPU_PMC + 20(r9)
  1041. BEGIN_FTR_SECTION
  1042. stw r10, VCPU_PMC + 24(r9)
  1043. stw r11, VCPU_PMC + 28(r9)
  1044. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  1045. 22:
  1046. /* Secondary threads go off to take a nap on POWER7 */
  1047. BEGIN_FTR_SECTION
  1048. lwz r0,VCPU_PTID(r9)
  1049. cmpwi r0,0
  1050. bne secondary_nap
  1051. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  1052. /* Restore host DABR and DABRX */
  1053. ld r5,HSTATE_DABR(r13)
  1054. li r6,7
  1055. mtspr SPRN_DABR,r5
  1056. mtspr SPRN_DABRX,r6
  1057. /* Restore SPRG3 */
  1058. ld r3,PACA_SPRG3(r13)
  1059. mtspr SPRN_SPRG3,r3
  1060. /*
  1061. * Reload DEC. HDEC interrupts were disabled when
  1062. * we reloaded the host's LPCR value.
  1063. */
  1064. ld r3, HSTATE_DECEXP(r13)
  1065. mftb r4
  1066. subf r4, r4, r3
  1067. mtspr SPRN_DEC, r4
  1068. /* Reload the host's PMU registers */
  1069. ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
  1070. lbz r4, LPPACA_PMCINUSE(r3)
  1071. cmpwi r4, 0
  1072. beq 23f /* skip if not */
  1073. lwz r3, HSTATE_PMC(r13)
  1074. lwz r4, HSTATE_PMC + 4(r13)
  1075. lwz r5, HSTATE_PMC + 8(r13)
  1076. lwz r6, HSTATE_PMC + 12(r13)
  1077. lwz r8, HSTATE_PMC + 16(r13)
  1078. lwz r9, HSTATE_PMC + 20(r13)
  1079. BEGIN_FTR_SECTION
  1080. lwz r10, HSTATE_PMC + 24(r13)
  1081. lwz r11, HSTATE_PMC + 28(r13)
  1082. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  1083. mtspr SPRN_PMC1, r3
  1084. mtspr SPRN_PMC2, r4
  1085. mtspr SPRN_PMC3, r5
  1086. mtspr SPRN_PMC4, r6
  1087. mtspr SPRN_PMC5, r8
  1088. mtspr SPRN_PMC6, r9
  1089. BEGIN_FTR_SECTION
  1090. mtspr SPRN_PMC7, r10
  1091. mtspr SPRN_PMC8, r11
  1092. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
  1093. ld r3, HSTATE_MMCR(r13)
  1094. ld r4, HSTATE_MMCR + 8(r13)
  1095. ld r5, HSTATE_MMCR + 16(r13)
  1096. mtspr SPRN_MMCR1, r4
  1097. mtspr SPRN_MMCRA, r5
  1098. mtspr SPRN_MMCR0, r3
  1099. isync
  1100. 23:
  1101. /*
  1102. * For external and machine check interrupts, we need
  1103. * to call the Linux handler to process the interrupt.
  1104. * We do that by jumping to absolute address 0x500 for
  1105. * external interrupts, or the machine_check_fwnmi label
  1106. * for machine checks (since firmware might have patched
  1107. * the vector area at 0x200). The [h]rfid at the end of the
  1108. * handler will return to the book3s_hv_interrupts.S code.
  1109. * For other interrupts we do the rfid to get back
  1110. * to the book3s_hv_interrupts.S code here.
  1111. */
  1112. ld r8, HSTATE_VMHANDLER(r13)
  1113. ld r7, HSTATE_HOST_MSR(r13)
  1114. cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
  1115. cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
  1116. BEGIN_FTR_SECTION
  1117. beq 11f
  1118. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
  1119. /* RFI into the highmem handler, or branch to interrupt handler */
  1120. mfmsr r6
  1121. li r0, MSR_RI
  1122. andc r6, r6, r0
  1123. mtmsrd r6, 1 /* Clear RI in MSR */
  1124. mtsrr0 r8
  1125. mtsrr1 r7
  1126. beqa 0x500 /* external interrupt (PPC970) */
  1127. beq cr1, 13f /* machine check */
  1128. RFI
  1129. /* On POWER7, we have external interrupts set to use HSRR0/1 */
  1130. 11: mtspr SPRN_HSRR0, r8
  1131. mtspr SPRN_HSRR1, r7
  1132. ba 0x500
  1133. 13: b machine_check_fwnmi
  1134. /*
  1135. * Check whether an HDSI is an HPTE not found fault or something else.
  1136. * If it is an HPTE not found fault that is due to the guest accessing
  1137. * a page that they have mapped but which we have paged out, then
  1138. * we continue on with the guest exit path. In all other cases,
  1139. * reflect the HDSI to the guest as a DSI.
  1140. */
  1141. kvmppc_hdsi:
  1142. mfspr r4, SPRN_HDAR
  1143. mfspr r6, SPRN_HDSISR
  1144. /* HPTE not found fault or protection fault? */
  1145. andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
  1146. beq 1f /* if not, send it to the guest */
  1147. andi. r0, r11, MSR_DR /* data relocation enabled? */
  1148. beq 3f
  1149. clrrdi r0, r4, 28
  1150. PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
  1151. bne 1f /* if no SLB entry found */
  1152. 4: std r4, VCPU_FAULT_DAR(r9)
  1153. stw r6, VCPU_FAULT_DSISR(r9)
  1154. /* Search the hash table. */
  1155. mr r3, r9 /* vcpu pointer */
  1156. li r7, 1 /* data fault */
  1157. bl .kvmppc_hpte_hv_fault
  1158. ld r9, HSTATE_KVM_VCPU(r13)
  1159. ld r10, VCPU_PC(r9)
  1160. ld r11, VCPU_MSR(r9)
  1161. li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
  1162. cmpdi r3, 0 /* retry the instruction */
  1163. beq 6f
  1164. cmpdi r3, -1 /* handle in kernel mode */
  1165. beq guest_exit_cont
  1166. cmpdi r3, -2 /* MMIO emulation; need instr word */
  1167. beq 2f
  1168. /* Synthesize a DSI for the guest */
  1169. ld r4, VCPU_FAULT_DAR(r9)
  1170. mr r6, r3
  1171. 1: mtspr SPRN_DAR, r4
  1172. mtspr SPRN_DSISR, r6
  1173. mtspr SPRN_SRR0, r10
  1174. mtspr SPRN_SRR1, r11
  1175. li r10, BOOK3S_INTERRUPT_DATA_STORAGE
  1176. li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
  1177. rotldi r11, r11, 63
  1178. fast_interrupt_c_return:
  1179. 6: ld r7, VCPU_CTR(r9)
  1180. lwz r8, VCPU_XER(r9)
  1181. mtctr r7
  1182. mtxer r8
  1183. mr r4, r9
  1184. b fast_guest_return
  1185. 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
  1186. ld r5, KVM_VRMA_SLB_V(r5)
  1187. b 4b
  1188. /* If this is for emulated MMIO, load the instruction word */
  1189. 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
  1190. /* Set guest mode to 'jump over instruction' so if lwz faults
  1191. * we'll just continue at the next IP. */
  1192. li r0, KVM_GUEST_MODE_SKIP
  1193. stb r0, HSTATE_IN_GUEST(r13)
  1194. /* Do the access with MSR:DR enabled */
  1195. mfmsr r3
  1196. ori r4, r3, MSR_DR /* Enable paging for data */
  1197. mtmsrd r4
  1198. lwz r8, 0(r10)
  1199. mtmsrd r3
  1200. /* Store the result */
  1201. stw r8, VCPU_LAST_INST(r9)
  1202. /* Unset guest mode. */
  1203. li r0, KVM_GUEST_MODE_NONE
  1204. stb r0, HSTATE_IN_GUEST(r13)
  1205. b guest_exit_cont
  1206. /*
  1207. * Similarly for an HISI, reflect it to the guest as an ISI unless
  1208. * it is an HPTE not found fault for a page that we have paged out.
  1209. */
  1210. kvmppc_hisi:
  1211. andis. r0, r11, SRR1_ISI_NOPT@h
  1212. beq 1f
  1213. andi. r0, r11, MSR_IR /* instruction relocation enabled? */
  1214. beq 3f
  1215. clrrdi r0, r10, 28
  1216. PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
  1217. bne 1f /* if no SLB entry found */
  1218. 4:
  1219. /* Search the hash table. */
  1220. mr r3, r9 /* vcpu pointer */
  1221. mr r4, r10
  1222. mr r6, r11
  1223. li r7, 0 /* instruction fault */
  1224. bl .kvmppc_hpte_hv_fault
  1225. ld r9, HSTATE_KVM_VCPU(r13)
  1226. ld r10, VCPU_PC(r9)
  1227. ld r11, VCPU_MSR(r9)
  1228. li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
  1229. cmpdi r3, 0 /* retry the instruction */
  1230. beq fast_interrupt_c_return
  1231. cmpdi r3, -1 /* handle in kernel mode */
  1232. beq guest_exit_cont
  1233. /* Synthesize an ISI for the guest */
  1234. mr r11, r3
  1235. 1: mtspr SPRN_SRR0, r10
  1236. mtspr SPRN_SRR1, r11
  1237. li r10, BOOK3S_INTERRUPT_INST_STORAGE
  1238. li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
  1239. rotldi r11, r11, 63
  1240. b fast_interrupt_c_return
  1241. 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
  1242. ld r5, KVM_VRMA_SLB_V(r6)
  1243. b 4b
  1244. /*
  1245. * Try to handle an hcall in real mode.
  1246. * Returns to the guest if we handle it, or continues on up to
  1247. * the kernel if we can't (i.e. if we don't have a handler for
  1248. * it, or if the handler returns H_TOO_HARD).
  1249. */
  1250. .globl hcall_try_real_mode
  1251. hcall_try_real_mode:
  1252. ld r3,VCPU_GPR(R3)(r9)
  1253. andi. r0,r11,MSR_PR
  1254. bne guest_exit_cont
  1255. clrrdi r3,r3,2
  1256. cmpldi r3,hcall_real_table_end - hcall_real_table
  1257. bge guest_exit_cont
  1258. LOAD_REG_ADDR(r4, hcall_real_table)
  1259. lwax r3,r3,r4
  1260. cmpwi r3,0
  1261. beq guest_exit_cont
  1262. add r3,r3,r4
  1263. mtctr r3
  1264. mr r3,r9 /* get vcpu pointer */
  1265. ld r4,VCPU_GPR(R4)(r9)
  1266. bctrl
  1267. cmpdi r3,H_TOO_HARD
  1268. beq hcall_real_fallback
  1269. ld r4,HSTATE_KVM_VCPU(r13)
  1270. std r3,VCPU_GPR(R3)(r4)
  1271. ld r10,VCPU_PC(r4)
  1272. ld r11,VCPU_MSR(r4)
  1273. b fast_guest_return
  1274. /* We've attempted a real mode hcall, but it's punted it back
  1275. * to userspace. We need to restore some clobbered volatiles
  1276. * before resuming the pass-it-to-qemu path */
  1277. hcall_real_fallback:
  1278. li r12,BOOK3S_INTERRUPT_SYSCALL
  1279. ld r9, HSTATE_KVM_VCPU(r13)
  1280. b guest_exit_cont
  1281. .globl hcall_real_table
  1282. hcall_real_table:
  1283. .long 0 /* 0 - unused */
  1284. .long .kvmppc_h_remove - hcall_real_table
  1285. .long .kvmppc_h_enter - hcall_real_table
  1286. .long .kvmppc_h_read - hcall_real_table
  1287. .long 0 /* 0x10 - H_CLEAR_MOD */
  1288. .long 0 /* 0x14 - H_CLEAR_REF */
  1289. .long .kvmppc_h_protect - hcall_real_table
  1290. .long 0 /* 0x1c - H_GET_TCE */
  1291. .long .kvmppc_h_put_tce - hcall_real_table
  1292. .long 0 /* 0x24 - H_SET_SPRG0 */
  1293. .long .kvmppc_h_set_dabr - hcall_real_table
  1294. .long 0 /* 0x2c */
  1295. .long 0 /* 0x30 */
  1296. .long 0 /* 0x34 */
  1297. .long 0 /* 0x38 */
  1298. .long 0 /* 0x3c */
  1299. .long 0 /* 0x40 */
  1300. .long 0 /* 0x44 */
  1301. .long 0 /* 0x48 */
  1302. .long 0 /* 0x4c */
  1303. .long 0 /* 0x50 */
  1304. .long 0 /* 0x54 */
  1305. .long 0 /* 0x58 */
  1306. .long 0 /* 0x5c */
  1307. .long 0 /* 0x60 */
  1308. #ifdef CONFIG_KVM_XICS
  1309. .long .kvmppc_rm_h_eoi - hcall_real_table
  1310. .long .kvmppc_rm_h_cppr - hcall_real_table
  1311. .long .kvmppc_rm_h_ipi - hcall_real_table
  1312. .long 0 /* 0x70 - H_IPOLL */
  1313. .long .kvmppc_rm_h_xirr - hcall_real_table
  1314. #else
  1315. .long 0 /* 0x64 - H_EOI */
  1316. .long 0 /* 0x68 - H_CPPR */
  1317. .long 0 /* 0x6c - H_IPI */
  1318. .long 0 /* 0x70 - H_IPOLL */
  1319. .long 0 /* 0x74 - H_XIRR */
  1320. #endif
  1321. .long 0 /* 0x78 */
  1322. .long 0 /* 0x7c */
  1323. .long 0 /* 0x80 */
  1324. .long 0 /* 0x84 */
  1325. .long 0 /* 0x88 */
  1326. .long 0 /* 0x8c */
  1327. .long 0 /* 0x90 */
  1328. .long 0 /* 0x94 */
  1329. .long 0 /* 0x98 */
  1330. .long 0 /* 0x9c */
  1331. .long 0 /* 0xa0 */
  1332. .long 0 /* 0xa4 */
  1333. .long 0 /* 0xa8 */
  1334. .long 0 /* 0xac */
  1335. .long 0 /* 0xb0 */
  1336. .long 0 /* 0xb4 */
  1337. .long 0 /* 0xb8 */
  1338. .long 0 /* 0xbc */
  1339. .long 0 /* 0xc0 */
  1340. .long 0 /* 0xc4 */
  1341. .long 0 /* 0xc8 */
  1342. .long 0 /* 0xcc */
  1343. .long 0 /* 0xd0 */
  1344. .long 0 /* 0xd4 */
  1345. .long 0 /* 0xd8 */
  1346. .long 0 /* 0xdc */
  1347. .long .kvmppc_h_cede - hcall_real_table
  1348. .long 0 /* 0xe4 */
  1349. .long 0 /* 0xe8 */
  1350. .long 0 /* 0xec */
  1351. .long 0 /* 0xf0 */
  1352. .long 0 /* 0xf4 */
  1353. .long 0 /* 0xf8 */
  1354. .long 0 /* 0xfc */
  1355. .long 0 /* 0x100 */
  1356. .long 0 /* 0x104 */
  1357. .long 0 /* 0x108 */
  1358. .long 0 /* 0x10c */
  1359. .long 0 /* 0x110 */
  1360. .long 0 /* 0x114 */
  1361. .long 0 /* 0x118 */
  1362. .long 0 /* 0x11c */
  1363. .long 0 /* 0x120 */
  1364. .long .kvmppc_h_bulk_remove - hcall_real_table
  1365. hcall_real_table_end:
  1366. ignore_hdec:
  1367. mr r4,r9
  1368. b fast_guest_return
  1369. _GLOBAL(kvmppc_h_set_dabr)
  1370. std r4,VCPU_DABR(r3)
  1371. /* Work around P7 bug where DABR can get corrupted on mtspr */
  1372. 1: mtspr SPRN_DABR,r4
  1373. mfspr r5, SPRN_DABR
  1374. cmpd r4, r5
  1375. bne 1b
  1376. isync
  1377. li r3,0
  1378. blr
  1379. _GLOBAL(kvmppc_h_cede)
  1380. ori r11,r11,MSR_EE
  1381. std r11,VCPU_MSR(r3)
  1382. li r0,1
  1383. stb r0,VCPU_CEDED(r3)
  1384. sync /* order setting ceded vs. testing prodded */
  1385. lbz r5,VCPU_PRODDED(r3)
  1386. cmpwi r5,0
  1387. bne kvm_cede_prodded
  1388. li r0,0 /* set trap to 0 to say hcall is handled */
  1389. stw r0,VCPU_TRAP(r3)
  1390. li r0,H_SUCCESS
  1391. std r0,VCPU_GPR(R3)(r3)
  1392. BEGIN_FTR_SECTION
  1393. b kvm_cede_exit /* just send it up to host on 970 */
  1394. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
  1395. /*
  1396. * Set our bit in the bitmask of napping threads unless all the
  1397. * other threads are already napping, in which case we send this
  1398. * up to the host.
  1399. */
  1400. ld r5,HSTATE_KVM_VCORE(r13)
  1401. lwz r6,VCPU_PTID(r3)
  1402. lwz r8,VCORE_ENTRY_EXIT(r5)
  1403. clrldi r8,r8,56
  1404. li r0,1
  1405. sld r0,r0,r6
  1406. addi r6,r5,VCORE_NAPPING_THREADS
  1407. 31: lwarx r4,0,r6
  1408. or r4,r4,r0
  1409. PPC_POPCNTW(R7,R4)
  1410. cmpw r7,r8
  1411. bge kvm_cede_exit
  1412. stwcx. r4,0,r6
  1413. bne 31b
  1414. li r0,1
  1415. stb r0,HSTATE_NAPPING(r13)
  1416. /* order napping_threads update vs testing entry_exit_count */
  1417. lwsync
  1418. mr r4,r3
  1419. lwz r7,VCORE_ENTRY_EXIT(r5)
  1420. cmpwi r7,0x100
  1421. bge 33f /* another thread already exiting */
  1422. /*
  1423. * Although not specifically required by the architecture, POWER7
  1424. * preserves the following registers in nap mode, even if an SMT mode
  1425. * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
  1426. * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
  1427. */
  1428. /* Save non-volatile GPRs */
  1429. std r14, VCPU_GPR(R14)(r3)
  1430. std r15, VCPU_GPR(R15)(r3)
  1431. std r16, VCPU_GPR(R16)(r3)
  1432. std r17, VCPU_GPR(R17)(r3)
  1433. std r18, VCPU_GPR(R18)(r3)
  1434. std r19, VCPU_GPR(R19)(r3)
  1435. std r20, VCPU_GPR(R20)(r3)
  1436. std r21, VCPU_GPR(R21)(r3)
  1437. std r22, VCPU_GPR(R22)(r3)
  1438. std r23, VCPU_GPR(R23)(r3)
  1439. std r24, VCPU_GPR(R24)(r3)
  1440. std r25, VCPU_GPR(R25)(r3)
  1441. std r26, VCPU_GPR(R26)(r3)
  1442. std r27, VCPU_GPR(R27)(r3)
  1443. std r28, VCPU_GPR(R28)(r3)
  1444. std r29, VCPU_GPR(R29)(r3)
  1445. std r30, VCPU_GPR(R30)(r3)
  1446. std r31, VCPU_GPR(R31)(r3)
  1447. /* save FP state */
  1448. bl .kvmppc_save_fp
  1449. /*
  1450. * Take a nap until a decrementer or external interrupt occurs,
  1451. * with PECE1 (wake on decr) and PECE0 (wake on external) set in LPCR
  1452. */
  1453. li r0,1
  1454. stb r0,HSTATE_HWTHREAD_REQ(r13)
  1455. mfspr r5,SPRN_LPCR
  1456. ori r5,r5,LPCR_PECE0 | LPCR_PECE1
  1457. mtspr SPRN_LPCR,r5
  1458. isync
  1459. li r0, 0
  1460. std r0, HSTATE_SCRATCH0(r13)
  1461. ptesync
  1462. ld r0, HSTATE_SCRATCH0(r13)
  1463. 1: cmpd r0, r0
  1464. bne 1b
  1465. nap
  1466. b .
  1467. kvm_end_cede:
  1468. /* get vcpu pointer */
  1469. ld r4, HSTATE_KVM_VCPU(r13)
  1470. /* Woken by external or decrementer interrupt */
  1471. ld r1, HSTATE_HOST_R1(r13)
  1472. /* load up FP state */
  1473. bl kvmppc_load_fp
  1474. /* Load NV GPRS */
  1475. ld r14, VCPU_GPR(R14)(r4)
  1476. ld r15, VCPU_GPR(R15)(r4)
  1477. ld r16, VCPU_GPR(R16)(r4)
  1478. ld r17, VCPU_GPR(R17)(r4)
  1479. ld r18, VCPU_GPR(R18)(r4)
  1480. ld r19, VCPU_GPR(R19)(r4)
  1481. ld r20, VCPU_GPR(R20)(r4)
  1482. ld r21, VCPU_GPR(R21)(r4)
  1483. ld r22, VCPU_GPR(R22)(r4)
  1484. ld r23, VCPU_GPR(R23)(r4)
  1485. ld r24, VCPU_GPR(R24)(r4)
  1486. ld r25, VCPU_GPR(R25)(r4)
  1487. ld r26, VCPU_GPR(R26)(r4)
  1488. ld r27, VCPU_GPR(R27)(r4)
  1489. ld r28, VCPU_GPR(R28)(r4)
  1490. ld r29, VCPU_GPR(R29)(r4)
  1491. ld r30, VCPU_GPR(R30)(r4)
  1492. ld r31, VCPU_GPR(R31)(r4)
  1493. /* clear our bit in vcore->napping_threads */
  1494. 33: ld r5,HSTATE_KVM_VCORE(r13)
  1495. lwz r3,VCPU_PTID(r4)
  1496. li r0,1
  1497. sld r0,r0,r3
  1498. addi r6,r5,VCORE_NAPPING_THREADS
  1499. 32: lwarx r7,0,r6
  1500. andc r7,r7,r0
  1501. stwcx. r7,0,r6
  1502. bne 32b
  1503. li r0,0
  1504. stb r0,HSTATE_NAPPING(r13)
  1505. /* Check the wake reason in SRR1 to see why we got here */
  1506. mfspr r3, SPRN_SRR1
  1507. rlwinm r3, r3, 44-31, 0x7 /* extract wake reason field */
  1508. cmpwi r3, 4 /* was it an external interrupt? */
  1509. li r12, BOOK3S_INTERRUPT_EXTERNAL
  1510. mr r9, r4
  1511. ld r10, VCPU_PC(r9)
  1512. ld r11, VCPU_MSR(r9)
  1513. beq do_ext_interrupt /* if so */
  1514. /* see if any other thread is already exiting */
  1515. lwz r0,VCORE_ENTRY_EXIT(r5)
  1516. cmpwi r0,0x100
  1517. blt kvmppc_cede_reentry /* if not go back to guest */
  1518. /* some threads are exiting, so go to the guest exit path */
  1519. b hcall_real_fallback
  1520. /* cede when already previously prodded case */
  1521. kvm_cede_prodded:
  1522. li r0,0
  1523. stb r0,VCPU_PRODDED(r3)
  1524. sync /* order testing prodded vs. clearing ceded */
  1525. stb r0,VCPU_CEDED(r3)
  1526. li r3,H_SUCCESS
  1527. blr
  1528. /* we've ceded but we want to give control to the host */
  1529. kvm_cede_exit:
  1530. b hcall_real_fallback
  1531. /* Try to handle a machine check in real mode */
  1532. machine_check_realmode:
  1533. mr r3, r9 /* get vcpu pointer */
  1534. bl .kvmppc_realmode_machine_check
  1535. nop
  1536. cmpdi r3, 0 /* continue exiting from guest? */
  1537. ld r9, HSTATE_KVM_VCPU(r13)
  1538. li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
  1539. beq mc_cont
  1540. /* If not, deliver a machine check. SRR0/1 are already set */
  1541. li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
  1542. li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */
  1543. rotldi r11, r11, 63
  1544. b fast_interrupt_c_return
  1545. secondary_too_late:
  1546. ld r5,HSTATE_KVM_VCORE(r13)
  1547. HMT_LOW
  1548. 13: lbz r3,VCORE_IN_GUEST(r5)
  1549. cmpwi r3,0
  1550. bne 13b
  1551. HMT_MEDIUM
  1552. ld r11,PACA_SLBSHADOWPTR(r13)
  1553. .rept SLB_NUM_BOLTED
  1554. ld r5,SLBSHADOW_SAVEAREA(r11)
  1555. ld r6,SLBSHADOW_SAVEAREA+8(r11)
  1556. andis. r7,r5,SLB_ESID_V@h
  1557. beq 1f
  1558. slbmte r6,r5
  1559. 1: addi r11,r11,16
  1560. .endr
  1561. secondary_nap:
  1562. /* Clear our vcpu pointer so we don't come back in early */
  1563. li r0, 0
  1564. std r0, HSTATE_KVM_VCPU(r13)
  1565. lwsync
  1566. /* Clear any pending IPI - assume we're a secondary thread */
  1567. ld r5, HSTATE_XICS_PHYS(r13)
  1568. li r7, XICS_XIRR
  1569. lwzcix r3, r5, r7 /* ack any pending interrupt */
  1570. rlwinm. r0, r3, 0, 0xffffff /* any pending? */
  1571. beq 37f
  1572. sync
  1573. li r0, 0xff
  1574. li r6, XICS_MFRR
  1575. stbcix r0, r5, r6 /* clear the IPI */
  1576. stwcix r3, r5, r7 /* EOI it */
  1577. 37: sync
  1578. /* increment the nap count and then go to nap mode */
  1579. ld r4, HSTATE_KVM_VCORE(r13)
  1580. addi r4, r4, VCORE_NAP_COUNT
  1581. lwsync /* make previous updates visible */
  1582. 51: lwarx r3, 0, r4
  1583. addi r3, r3, 1
  1584. stwcx. r3, 0, r4
  1585. bne 51b
  1586. kvm_no_guest:
  1587. li r0, KVM_HWTHREAD_IN_NAP
  1588. stb r0, HSTATE_HWTHREAD_STATE(r13)
  1589. li r3, LPCR_PECE0
  1590. mfspr r4, SPRN_LPCR
  1591. rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
  1592. mtspr SPRN_LPCR, r4
  1593. isync
  1594. std r0, HSTATE_SCRATCH0(r13)
  1595. ptesync
  1596. ld r0, HSTATE_SCRATCH0(r13)
  1597. 1: cmpd r0, r0
  1598. bne 1b
  1599. nap
  1600. b .
  1601. /*
  1602. * Save away FP, VMX and VSX registers.
  1603. * r3 = vcpu pointer
  1604. */
  1605. _GLOBAL(kvmppc_save_fp)
  1606. mfmsr r5
  1607. ori r8,r5,MSR_FP
  1608. #ifdef CONFIG_ALTIVEC
  1609. BEGIN_FTR_SECTION
  1610. oris r8,r8,MSR_VEC@h
  1611. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  1612. #endif
  1613. #ifdef CONFIG_VSX
  1614. BEGIN_FTR_SECTION
  1615. oris r8,r8,MSR_VSX@h
  1616. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  1617. #endif
  1618. mtmsrd r8
  1619. isync
  1620. #ifdef CONFIG_VSX
  1621. BEGIN_FTR_SECTION
  1622. reg = 0
  1623. .rept 32
  1624. li r6,reg*16+VCPU_VSRS
  1625. STXVD2X(reg,R6,R3)
  1626. reg = reg + 1
  1627. .endr
  1628. FTR_SECTION_ELSE
  1629. #endif
  1630. reg = 0
  1631. .rept 32
  1632. stfd reg,reg*8+VCPU_FPRS(r3)
  1633. reg = reg + 1
  1634. .endr
  1635. #ifdef CONFIG_VSX
  1636. ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
  1637. #endif
  1638. mffs fr0
  1639. stfd fr0,VCPU_FPSCR(r3)
  1640. #ifdef CONFIG_ALTIVEC
  1641. BEGIN_FTR_SECTION
  1642. reg = 0
  1643. .rept 32
  1644. li r6,reg*16+VCPU_VRS
  1645. stvx reg,r6,r3
  1646. reg = reg + 1
  1647. .endr
  1648. mfvscr vr0
  1649. li r6,VCPU_VSCR
  1650. stvx vr0,r6,r3
  1651. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  1652. #endif
  1653. mfspr r6,SPRN_VRSAVE
  1654. stw r6,VCPU_VRSAVE(r3)
  1655. mtmsrd r5
  1656. isync
  1657. blr
  1658. /*
  1659. * Load up FP, VMX and VSX registers
  1660. * r4 = vcpu pointer
  1661. */
  1662. .globl kvmppc_load_fp
  1663. kvmppc_load_fp:
  1664. mfmsr r9
  1665. ori r8,r9,MSR_FP
  1666. #ifdef CONFIG_ALTIVEC
  1667. BEGIN_FTR_SECTION
  1668. oris r8,r8,MSR_VEC@h
  1669. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  1670. #endif
  1671. #ifdef CONFIG_VSX
  1672. BEGIN_FTR_SECTION
  1673. oris r8,r8,MSR_VSX@h
  1674. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  1675. #endif
  1676. mtmsrd r8
  1677. isync
  1678. lfd fr0,VCPU_FPSCR(r4)
  1679. MTFSF_L(fr0)
  1680. #ifdef CONFIG_VSX
  1681. BEGIN_FTR_SECTION
  1682. reg = 0
  1683. .rept 32
  1684. li r7,reg*16+VCPU_VSRS
  1685. LXVD2X(reg,R7,R4)
  1686. reg = reg + 1
  1687. .endr
  1688. FTR_SECTION_ELSE
  1689. #endif
  1690. reg = 0
  1691. .rept 32
  1692. lfd reg,reg*8+VCPU_FPRS(r4)
  1693. reg = reg + 1
  1694. .endr
  1695. #ifdef CONFIG_VSX
  1696. ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
  1697. #endif
  1698. #ifdef CONFIG_ALTIVEC
  1699. BEGIN_FTR_SECTION
  1700. li r7,VCPU_VSCR
  1701. lvx vr0,r7,r4
  1702. mtvscr vr0
  1703. reg = 0
  1704. .rept 32
  1705. li r7,reg*16+VCPU_VRS
  1706. lvx reg,r7,r4
  1707. reg = reg + 1
  1708. .endr
  1709. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  1710. #endif
  1711. lwz r7,VCPU_VRSAVE(r4)
  1712. mtspr SPRN_VRSAVE,r7
  1713. blr