exceptions-64s.S 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512
  1. /*
  2. * This file contains the 64-bit "server" PowerPC variant
  3. * of the low level exception handling including exception
  4. * vectors, exception return, part of the slb and stab
  5. * handling and other fixed offset specific things.
  6. *
  7. * This file is meant to be #included from head_64.S due to
  8. * position dependent assembly.
  9. *
  10. * Most of this originates from head_64.S and thus has the same
  11. * copyright history.
  12. *
  13. */
  14. #include <asm/hw_irq.h>
  15. #include <asm/exception-64s.h>
  16. #include <asm/ptrace.h>
  17. /*
  18. * We layout physical memory as follows:
  19. * 0x0000 - 0x00ff : Secondary processor spin code
  20. * 0x0100 - 0x17ff : pSeries Interrupt prologs
  21. * 0x1800 - 0x4000 : interrupt support common interrupt prologs
  22. * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
  23. * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
  24. * 0x7000 - 0x7fff : FWNMI data area
  25. * 0x8000 - 0x8fff : Initial (CPU0) segment table
  26. * 0x9000 - : Early init and support code
  27. */
  28. /* Syscall routine is used twice, in reloc-off and reloc-on paths */
  29. #define SYSCALL_PSERIES_1 \
  30. BEGIN_FTR_SECTION \
  31. cmpdi r0,0x1ebe ; \
  32. beq- 1f ; \
  33. END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \
  34. mr r9,r13 ; \
  35. GET_PACA(r13) ; \
  36. mfspr r11,SPRN_SRR0 ; \
  37. 0:
  38. #define SYSCALL_PSERIES_2_RFID \
  39. mfspr r12,SPRN_SRR1 ; \
  40. ld r10,PACAKBASE(r13) ; \
  41. LOAD_HANDLER(r10, system_call_entry) ; \
  42. mtspr SPRN_SRR0,r10 ; \
  43. ld r10,PACAKMSR(r13) ; \
  44. mtspr SPRN_SRR1,r10 ; \
  45. rfid ; \
  46. b . ; /* prevent speculative execution */
  47. #define SYSCALL_PSERIES_3 \
  48. /* Fast LE/BE switch system call */ \
  49. 1: mfspr r12,SPRN_SRR1 ; \
  50. xori r12,r12,MSR_LE ; \
  51. mtspr SPRN_SRR1,r12 ; \
  52. rfid ; /* return to userspace */ \
  53. b . ; \
  54. 2: mfspr r12,SPRN_SRR1 ; \
  55. andi. r12,r12,MSR_PR ; \
  56. bne 0b ; \
  57. mtspr SPRN_SRR0,r3 ; \
  58. mtspr SPRN_SRR1,r4 ; \
  59. mtspr SPRN_SDR1,r5 ; \
  60. rfid ; \
  61. b . ; /* prevent speculative execution */
  62. #if defined(CONFIG_RELOCATABLE)
  63. /*
  64. * We can't branch directly; in the direct case we use LR
  65. * and system_call_entry restores LR. (We thus need to move
  66. * LR to r10 in the RFID case too.)
  67. */
  68. #define SYSCALL_PSERIES_2_DIRECT \
  69. mflr r10 ; \
  70. ld r12,PACAKBASE(r13) ; \
  71. LOAD_HANDLER(r12, system_call_entry_direct) ; \
  72. mtctr r12 ; \
  73. mfspr r12,SPRN_SRR1 ; \
  74. /* Re-use of r13... No spare regs to do this */ \
  75. li r13,MSR_RI ; \
  76. mtmsrd r13,1 ; \
  77. GET_PACA(r13) ; /* get r13 back */ \
  78. bctr ;
  79. #else
  80. /* We can branch directly */
  81. #define SYSCALL_PSERIES_2_DIRECT \
  82. mfspr r12,SPRN_SRR1 ; \
  83. li r10,MSR_RI ; \
  84. mtmsrd r10,1 ; /* Set RI (EE=0) */ \
  85. b system_call_entry_direct ;
  86. #endif
  87. /*
  88. * This is the start of the interrupt handlers for pSeries
  89. * This code runs with relocation off.
  90. * Code from here to __end_interrupts gets copied down to real
  91. * address 0x100 when we are running a relocatable kernel.
  92. * Therefore any relative branches in this section must only
  93. * branch to labels in this section.
  94. */
  95. . = 0x100
  96. .globl __start_interrupts
  97. __start_interrupts:
  98. .globl system_reset_pSeries;
  99. system_reset_pSeries:
  100. HMT_MEDIUM_PPR_DISCARD
  101. SET_SCRATCH0(r13)
  102. #ifdef CONFIG_PPC_P7_NAP
  103. BEGIN_FTR_SECTION
  104. /* Running native on arch 2.06 or later, check if we are
  105. * waking up from nap. We only handle no state loss and
  106. * supervisor state loss. We do -not- handle hypervisor
  107. * state loss at this time.
  108. */
  109. mfspr r13,SPRN_SRR1
  110. rlwinm. r13,r13,47-31,30,31
  111. beq 9f
  112. /* waking up from powersave (nap) state */
  113. cmpwi cr1,r13,2
  114. /* Total loss of HV state is fatal, we could try to use the
  115. * PIR to locate a PACA, then use an emergency stack etc...
  116. * but for now, let's just stay stuck here
  117. */
  118. bgt cr1,.
  119. GET_PACA(r13)
  120. #ifdef CONFIG_KVM_BOOK3S_64_HV
  121. li r0,KVM_HWTHREAD_IN_KERNEL
  122. stb r0,HSTATE_HWTHREAD_STATE(r13)
  123. /* Order setting hwthread_state vs. testing hwthread_req */
  124. sync
  125. lbz r0,HSTATE_HWTHREAD_REQ(r13)
  126. cmpwi r0,0
  127. beq 1f
  128. b kvm_start_guest
  129. 1:
  130. #endif
  131. beq cr1,2f
  132. b .power7_wakeup_noloss
  133. 2: b .power7_wakeup_loss
  134. 9:
  135. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  136. #endif /* CONFIG_PPC_P7_NAP */
  137. EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
  138. NOTEST, 0x100)
  139. . = 0x200
  140. machine_check_pSeries_1:
  141. /* This is moved out of line as it can be patched by FW, but
  142. * some code path might still want to branch into the original
  143. * vector
  144. */
  145. HMT_MEDIUM_PPR_DISCARD
  146. SET_SCRATCH0(r13) /* save r13 */
  147. EXCEPTION_PROLOG_0(PACA_EXMC)
  148. b machine_check_pSeries_0
  149. . = 0x300
  150. .globl data_access_pSeries
  151. data_access_pSeries:
  152. HMT_MEDIUM_PPR_DISCARD
  153. SET_SCRATCH0(r13)
  154. BEGIN_FTR_SECTION
  155. b data_access_check_stab
  156. data_access_not_stab:
  157. END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
  158. EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
  159. KVMTEST, 0x300)
  160. . = 0x380
  161. .globl data_access_slb_pSeries
  162. data_access_slb_pSeries:
  163. HMT_MEDIUM_PPR_DISCARD
  164. SET_SCRATCH0(r13)
  165. EXCEPTION_PROLOG_0(PACA_EXSLB)
  166. EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
  167. std r3,PACA_EXSLB+EX_R3(r13)
  168. mfspr r3,SPRN_DAR
  169. #ifdef __DISABLED__
  170. /* Keep that around for when we re-implement dynamic VSIDs */
  171. cmpdi r3,0
  172. bge slb_miss_user_pseries
  173. #endif /* __DISABLED__ */
  174. mfspr r12,SPRN_SRR1
  175. #ifndef CONFIG_RELOCATABLE
  176. b .slb_miss_realmode
  177. #else
  178. /*
  179. * We can't just use a direct branch to .slb_miss_realmode
  180. * because the distance from here to there depends on where
  181. * the kernel ends up being put.
  182. */
  183. mfctr r11
  184. ld r10,PACAKBASE(r13)
  185. LOAD_HANDLER(r10, .slb_miss_realmode)
  186. mtctr r10
  187. bctr
  188. #endif
  189. STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
  190. . = 0x480
  191. .globl instruction_access_slb_pSeries
  192. instruction_access_slb_pSeries:
  193. HMT_MEDIUM_PPR_DISCARD
  194. SET_SCRATCH0(r13)
  195. EXCEPTION_PROLOG_0(PACA_EXSLB)
  196. EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
  197. std r3,PACA_EXSLB+EX_R3(r13)
  198. mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
  199. #ifdef __DISABLED__
  200. /* Keep that around for when we re-implement dynamic VSIDs */
  201. cmpdi r3,0
  202. bge slb_miss_user_pseries
  203. #endif /* __DISABLED__ */
  204. mfspr r12,SPRN_SRR1
  205. #ifndef CONFIG_RELOCATABLE
  206. b .slb_miss_realmode
  207. #else
  208. mfctr r11
  209. ld r10,PACAKBASE(r13)
  210. LOAD_HANDLER(r10, .slb_miss_realmode)
  211. mtctr r10
  212. bctr
  213. #endif
  214. /* We open code these as we can't have a ". = x" (even with
  215. * x = "." within a feature section
  216. */
  217. . = 0x500;
  218. .globl hardware_interrupt_pSeries;
  219. .globl hardware_interrupt_hv;
  220. hardware_interrupt_pSeries:
  221. hardware_interrupt_hv:
  222. HMT_MEDIUM_PPR_DISCARD
  223. BEGIN_FTR_SECTION
  224. _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
  225. EXC_HV, SOFTEN_TEST_HV)
  226. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
  227. FTR_SECTION_ELSE
  228. _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
  229. EXC_STD, SOFTEN_TEST_HV_201)
  230. KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
  231. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  232. STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
  233. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
  234. STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
  235. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
  236. STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
  237. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
  238. . = 0x900
  239. .globl decrementer_pSeries
  240. decrementer_pSeries:
  241. _MASKABLE_EXCEPTION_PSERIES(0x900, decrementer, EXC_STD, SOFTEN_TEST_PR)
  242. STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
  243. MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
  244. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
  245. STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
  246. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
  247. . = 0xc00
  248. .globl system_call_pSeries
  249. system_call_pSeries:
  250. HMT_MEDIUM
  251. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  252. SET_SCRATCH0(r13)
  253. GET_PACA(r13)
  254. std r9,PACA_EXGEN+EX_R9(r13)
  255. std r10,PACA_EXGEN+EX_R10(r13)
  256. mfcr r9
  257. KVMTEST(0xc00)
  258. GET_SCRATCH0(r13)
  259. #endif
  260. SYSCALL_PSERIES_1
  261. SYSCALL_PSERIES_2_RFID
  262. SYSCALL_PSERIES_3
  263. KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
  264. STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
  265. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
  266. /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
  267. * out of line to handle them
  268. */
  269. . = 0xe00
  270. hv_exception_trampoline:
  271. SET_SCRATCH0(r13)
  272. EXCEPTION_PROLOG_0(PACA_EXGEN)
  273. b h_data_storage_hv
  274. . = 0xe20
  275. SET_SCRATCH0(r13)
  276. EXCEPTION_PROLOG_0(PACA_EXGEN)
  277. b h_instr_storage_hv
  278. . = 0xe40
  279. SET_SCRATCH0(r13)
  280. EXCEPTION_PROLOG_0(PACA_EXGEN)
  281. b emulation_assist_hv
  282. . = 0xe60
  283. SET_SCRATCH0(r13)
  284. EXCEPTION_PROLOG_0(PACA_EXGEN)
  285. b hmi_exception_hv
  286. . = 0xe80
  287. SET_SCRATCH0(r13)
  288. EXCEPTION_PROLOG_0(PACA_EXGEN)
  289. b h_doorbell_hv
  290. /* We need to deal with the Altivec unavailable exception
  291. * here which is at 0xf20, thus in the middle of the
  292. * prolog code of the PerformanceMonitor one. A little
  293. * trickery is thus necessary
  294. */
  295. performance_monitor_pSeries_1:
  296. . = 0xf00
  297. SET_SCRATCH0(r13)
  298. EXCEPTION_PROLOG_0(PACA_EXGEN)
  299. b performance_monitor_pSeries
  300. altivec_unavailable_pSeries_1:
  301. . = 0xf20
  302. SET_SCRATCH0(r13)
  303. EXCEPTION_PROLOG_0(PACA_EXGEN)
  304. b altivec_unavailable_pSeries
  305. vsx_unavailable_pSeries_1:
  306. . = 0xf40
  307. SET_SCRATCH0(r13)
  308. EXCEPTION_PROLOG_0(PACA_EXGEN)
  309. b vsx_unavailable_pSeries
  310. facility_unavailable_trampoline:
  311. . = 0xf60
  312. SET_SCRATCH0(r13)
  313. EXCEPTION_PROLOG_0(PACA_EXGEN)
  314. b facility_unavailable_pSeries
  315. hv_facility_unavailable_trampoline:
  316. . = 0xf80
  317. SET_SCRATCH0(r13)
  318. EXCEPTION_PROLOG_0(PACA_EXGEN)
  319. b facility_unavailable_hv
  320. #ifdef CONFIG_CBE_RAS
  321. STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
  322. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
  323. #endif /* CONFIG_CBE_RAS */
  324. STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
  325. KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
  326. . = 0x1500
  327. .global denorm_exception_hv
  328. denorm_exception_hv:
  329. HMT_MEDIUM_PPR_DISCARD
  330. mtspr SPRN_SPRG_HSCRATCH0,r13
  331. EXCEPTION_PROLOG_0(PACA_EXGEN)
  332. std r11,PACA_EXGEN+EX_R11(r13)
  333. std r12,PACA_EXGEN+EX_R12(r13)
  334. mfspr r9,SPRN_SPRG_HSCRATCH0
  335. std r9,PACA_EXGEN+EX_R13(r13)
  336. mfcr r9
  337. #ifdef CONFIG_PPC_DENORMALISATION
  338. mfspr r10,SPRN_HSRR1
  339. mfspr r11,SPRN_HSRR0 /* save HSRR0 */
  340. andis. r10,r10,(HSRR1_DENORM)@h /* denorm? */
  341. addi r11,r11,-4 /* HSRR0 is next instruction */
  342. bne+ denorm_assist
  343. #endif
  344. EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
  345. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
  346. #ifdef CONFIG_CBE_RAS
  347. STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
  348. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
  349. #endif /* CONFIG_CBE_RAS */
  350. STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
  351. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
  352. #ifdef CONFIG_CBE_RAS
  353. STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
  354. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
  355. #else
  356. . = 0x1800
  357. #endif /* CONFIG_CBE_RAS */
  358. /*** Out of line interrupts support ***/
  359. .align 7
  360. /* moved from 0x200 */
  361. machine_check_pSeries:
  362. .globl machine_check_fwnmi
  363. machine_check_fwnmi:
  364. HMT_MEDIUM_PPR_DISCARD
  365. SET_SCRATCH0(r13) /* save r13 */
  366. EXCEPTION_PROLOG_0(PACA_EXMC)
  367. machine_check_pSeries_0:
  368. EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST, 0x200)
  369. EXCEPTION_PROLOG_PSERIES_1(machine_check_common, EXC_STD)
  370. KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
  371. /* moved from 0x300 */
  372. data_access_check_stab:
  373. GET_PACA(r13)
  374. std r9,PACA_EXSLB+EX_R9(r13)
  375. std r10,PACA_EXSLB+EX_R10(r13)
  376. mfspr r10,SPRN_DAR
  377. mfspr r9,SPRN_DSISR
  378. srdi r10,r10,60
  379. rlwimi r10,r9,16,0x20
  380. #ifdef CONFIG_KVM_BOOK3S_PR
  381. lbz r9,HSTATE_IN_GUEST(r13)
  382. rlwimi r10,r9,8,0x300
  383. #endif
  384. mfcr r9
  385. cmpwi r10,0x2c
  386. beq do_stab_bolted_pSeries
  387. mtcrf 0x80,r9
  388. ld r9,PACA_EXSLB+EX_R9(r13)
  389. ld r10,PACA_EXSLB+EX_R10(r13)
  390. b data_access_not_stab
  391. do_stab_bolted_pSeries:
  392. std r11,PACA_EXSLB+EX_R11(r13)
  393. std r12,PACA_EXSLB+EX_R12(r13)
  394. GET_SCRATCH0(r10)
  395. std r10,PACA_EXSLB+EX_R13(r13)
  396. EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
  397. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
  398. KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
  399. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
  400. KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
  401. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
  402. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
  403. #ifdef CONFIG_PPC_DENORMALISATION
  404. denorm_assist:
  405. BEGIN_FTR_SECTION
  406. /*
  407. * To denormalise we need to move a copy of the register to itself.
  408. * For POWER6 do that here for all FP regs.
  409. */
  410. mfmsr r10
  411. ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
  412. xori r10,r10,(MSR_FE0|MSR_FE1)
  413. mtmsrd r10
  414. sync
  415. #define FMR2(n) fmr (n), (n) ; fmr n+1, n+1
  416. #define FMR4(n) FMR2(n) ; FMR2(n+2)
  417. #define FMR8(n) FMR4(n) ; FMR4(n+4)
  418. #define FMR16(n) FMR8(n) ; FMR8(n+8)
  419. #define FMR32(n) FMR16(n) ; FMR16(n+16)
  420. FMR32(0)
  421. FTR_SECTION_ELSE
  422. /*
  423. * To denormalise we need to move a copy of the register to itself.
  424. * For POWER7 do that here for the first 32 VSX registers only.
  425. */
  426. mfmsr r10
  427. oris r10,r10,MSR_VSX@h
  428. mtmsrd r10
  429. sync
  430. #define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
  431. #define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
  432. #define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
  433. #define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
  434. #define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
  435. XVCPSGNDP32(0)
  436. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
  437. BEGIN_FTR_SECTION
  438. b denorm_done
  439. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  440. /*
  441. * To denormalise we need to move a copy of the register to itself.
  442. * For POWER8 we need to do that for all 64 VSX registers
  443. */
  444. XVCPSGNDP32(32)
  445. denorm_done:
  446. mtspr SPRN_HSRR0,r11
  447. mtcrf 0x80,r9
  448. ld r9,PACA_EXGEN+EX_R9(r13)
  449. RESTORE_PPR_PACA(PACA_EXGEN, r10)
  450. ld r10,PACA_EXGEN+EX_R10(r13)
  451. ld r11,PACA_EXGEN+EX_R11(r13)
  452. ld r12,PACA_EXGEN+EX_R12(r13)
  453. ld r13,PACA_EXGEN+EX_R13(r13)
  454. HRFID
  455. b .
  456. #endif
  457. .align 7
  458. /* moved from 0xe00 */
  459. STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
  460. KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
  461. STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
  462. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
  463. STD_EXCEPTION_HV_OOL(0xe42, emulation_assist)
  464. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
  465. STD_EXCEPTION_HV_OOL(0xe62, hmi_exception) /* need to flush cache ? */
  466. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
  467. MASKABLE_EXCEPTION_HV_OOL(0xe82, h_doorbell)
  468. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
  469. /* moved from 0xf00 */
  470. STD_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
  471. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
  472. STD_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
  473. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
  474. STD_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
  475. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
  476. STD_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
  477. KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf60)
  478. STD_EXCEPTION_HV_OOL(0xf82, facility_unavailable)
  479. KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xf82)
  480. /*
  481. * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
  482. * - If it was a decrementer interrupt, we bump the dec to max and and return.
  483. * - If it was a doorbell we return immediately since doorbells are edge
  484. * triggered and won't automatically refire.
  485. * - else we hard disable and return.
  486. * This is called with r10 containing the value to OR to the paca field.
  487. */
  488. #define MASKED_INTERRUPT(_H) \
  489. masked_##_H##interrupt: \
  490. std r11,PACA_EXGEN+EX_R11(r13); \
  491. lbz r11,PACAIRQHAPPENED(r13); \
  492. or r11,r11,r10; \
  493. stb r11,PACAIRQHAPPENED(r13); \
  494. cmpwi r10,PACA_IRQ_DEC; \
  495. bne 1f; \
  496. lis r10,0x7fff; \
  497. ori r10,r10,0xffff; \
  498. mtspr SPRN_DEC,r10; \
  499. b 2f; \
  500. 1: cmpwi r10,PACA_IRQ_DBELL; \
  501. beq 2f; \
  502. mfspr r10,SPRN_##_H##SRR1; \
  503. rldicl r10,r10,48,1; /* clear MSR_EE */ \
  504. rotldi r10,r10,16; \
  505. mtspr SPRN_##_H##SRR1,r10; \
  506. 2: mtcrf 0x80,r9; \
  507. ld r9,PACA_EXGEN+EX_R9(r13); \
  508. ld r10,PACA_EXGEN+EX_R10(r13); \
  509. ld r11,PACA_EXGEN+EX_R11(r13); \
  510. GET_SCRATCH0(r13); \
  511. ##_H##rfid; \
  512. b .
  513. MASKED_INTERRUPT()
  514. MASKED_INTERRUPT(H)
  515. /*
  516. * Called from arch_local_irq_enable when an interrupt needs
  517. * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
  518. * which kind of interrupt. MSR:EE is already off. We generate a
  519. * stackframe like if a real interrupt had happened.
  520. *
  521. * Note: While MSR:EE is off, we need to make sure that _MSR
  522. * in the generated frame has EE set to 1 or the exception
  523. * handler will not properly re-enable them.
  524. */
  525. _GLOBAL(__replay_interrupt)
  526. /* We are going to jump to the exception common code which
  527. * will retrieve various register values from the PACA which
  528. * we don't give a damn about, so we don't bother storing them.
  529. */
  530. mfmsr r12
  531. mflr r11
  532. mfcr r9
  533. ori r12,r12,MSR_EE
  534. cmpwi r3,0x900
  535. beq decrementer_common
  536. cmpwi r3,0x500
  537. beq hardware_interrupt_common
  538. BEGIN_FTR_SECTION
  539. cmpwi r3,0xe80
  540. beq h_doorbell_common
  541. FTR_SECTION_ELSE
  542. cmpwi r3,0xa00
  543. beq doorbell_super_common
  544. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
  545. blr
  546. #ifdef CONFIG_PPC_PSERIES
  547. /*
  548. * Vectors for the FWNMI option. Share common code.
  549. */
  550. .globl system_reset_fwnmi
  551. .align 7
  552. system_reset_fwnmi:
  553. HMT_MEDIUM_PPR_DISCARD
  554. SET_SCRATCH0(r13) /* save r13 */
  555. EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
  556. NOTEST, 0x100)
  557. #endif /* CONFIG_PPC_PSERIES */
  558. #ifdef __DISABLED__
  559. /*
  560. * This is used for when the SLB miss handler has to go virtual,
  561. * which doesn't happen for now anymore but will once we re-implement
  562. * dynamic VSIDs for shared page tables
  563. */
  564. slb_miss_user_pseries:
  565. std r10,PACA_EXGEN+EX_R10(r13)
  566. std r11,PACA_EXGEN+EX_R11(r13)
  567. std r12,PACA_EXGEN+EX_R12(r13)
  568. GET_SCRATCH0(r10)
  569. ld r11,PACA_EXSLB+EX_R9(r13)
  570. ld r12,PACA_EXSLB+EX_R3(r13)
  571. std r10,PACA_EXGEN+EX_R13(r13)
  572. std r11,PACA_EXGEN+EX_R9(r13)
  573. std r12,PACA_EXGEN+EX_R3(r13)
  574. clrrdi r12,r13,32
  575. mfmsr r10
  576. mfspr r11,SRR0 /* save SRR0 */
  577. ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
  578. ori r10,r10,MSR_IR|MSR_DR|MSR_RI
  579. mtspr SRR0,r12
  580. mfspr r12,SRR1 /* and SRR1 */
  581. mtspr SRR1,r10
  582. rfid
  583. b . /* prevent spec. execution */
  584. #endif /* __DISABLED__ */
  585. /*
  586. * Code from here down to __end_handlers is invoked from the
  587. * exception prologs above. Because the prologs assemble the
  588. * addresses of these handlers using the LOAD_HANDLER macro,
  589. * which uses an ori instruction, these handlers must be in
  590. * the first 64k of the kernel image.
  591. */
  592. /*** Common interrupt handlers ***/
  593. STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
  594. /*
  595. * Machine check is different because we use a different
  596. * save area: PACA_EXMC instead of PACA_EXGEN.
  597. */
  598. .align 7
  599. .globl machine_check_common
  600. machine_check_common:
  601. mfspr r10,SPRN_DAR
  602. std r10,PACA_EXGEN+EX_DAR(r13)
  603. mfspr r10,SPRN_DSISR
  604. stw r10,PACA_EXGEN+EX_DSISR(r13)
  605. EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
  606. FINISH_NAP
  607. DISABLE_INTS
  608. ld r3,PACA_EXGEN+EX_DAR(r13)
  609. lwz r4,PACA_EXGEN+EX_DSISR(r13)
  610. std r3,_DAR(r1)
  611. std r4,_DSISR(r1)
  612. bl .save_nvgprs
  613. addi r3,r1,STACK_FRAME_OVERHEAD
  614. bl .machine_check_exception
  615. b .ret_from_except
  616. STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
  617. STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt)
  618. STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt)
  619. #ifdef CONFIG_PPC_DOORBELL
  620. STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .doorbell_exception)
  621. #else
  622. STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .unknown_exception)
  623. #endif
  624. STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
  625. STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
  626. STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
  627. STD_EXCEPTION_COMMON(0xe40, emulation_assist, .emulation_assist_interrupt)
  628. STD_EXCEPTION_COMMON(0xe60, hmi_exception, .unknown_exception)
  629. #ifdef CONFIG_PPC_DOORBELL
  630. STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .doorbell_exception)
  631. #else
  632. STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .unknown_exception)
  633. #endif
  634. STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, .performance_monitor_exception)
  635. STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
  636. STD_EXCEPTION_COMMON(0x1502, denorm, .unknown_exception)
  637. #ifdef CONFIG_ALTIVEC
  638. STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
  639. #else
  640. STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
  641. #endif
  642. #ifdef CONFIG_CBE_RAS
  643. STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
  644. STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
  645. STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
  646. #endif /* CONFIG_CBE_RAS */
  647. /*
  648. * Relocation-on interrupts: A subset of the interrupts can be delivered
  649. * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
  650. * it. Addresses are the same as the original interrupt addresses, but
  651. * offset by 0xc000000000004000.
  652. * It's impossible to receive interrupts below 0x300 via this mechanism.
  653. * KVM: None of these traps are from the guest ; anything that escalated
  654. * to HV=1 from HV=0 is delivered via real mode handlers.
  655. */
  656. /*
  657. * This uses the standard macro, since the original 0x300 vector
  658. * only has extra guff for STAB-based processors -- which never
  659. * come here.
  660. */
  661. STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
  662. . = 0x4380
  663. .globl data_access_slb_relon_pSeries
  664. data_access_slb_relon_pSeries:
  665. SET_SCRATCH0(r13)
  666. EXCEPTION_PROLOG_0(PACA_EXSLB)
  667. EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
  668. std r3,PACA_EXSLB+EX_R3(r13)
  669. mfspr r3,SPRN_DAR
  670. mfspr r12,SPRN_SRR1
  671. #ifndef CONFIG_RELOCATABLE
  672. b .slb_miss_realmode
  673. #else
  674. /*
  675. * We can't just use a direct branch to .slb_miss_realmode
  676. * because the distance from here to there depends on where
  677. * the kernel ends up being put.
  678. */
  679. mfctr r11
  680. ld r10,PACAKBASE(r13)
  681. LOAD_HANDLER(r10, .slb_miss_realmode)
  682. mtctr r10
  683. bctr
  684. #endif
  685. STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
  686. . = 0x4480
  687. .globl instruction_access_slb_relon_pSeries
  688. instruction_access_slb_relon_pSeries:
  689. SET_SCRATCH0(r13)
  690. EXCEPTION_PROLOG_0(PACA_EXSLB)
  691. EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
  692. std r3,PACA_EXSLB+EX_R3(r13)
  693. mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
  694. mfspr r12,SPRN_SRR1
  695. #ifndef CONFIG_RELOCATABLE
  696. b .slb_miss_realmode
  697. #else
  698. mfctr r11
  699. ld r10,PACAKBASE(r13)
  700. LOAD_HANDLER(r10, .slb_miss_realmode)
  701. mtctr r10
  702. bctr
  703. #endif
  704. . = 0x4500
  705. .globl hardware_interrupt_relon_pSeries;
  706. .globl hardware_interrupt_relon_hv;
  707. hardware_interrupt_relon_pSeries:
  708. hardware_interrupt_relon_hv:
  709. BEGIN_FTR_SECTION
  710. _MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
  711. FTR_SECTION_ELSE
  712. _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
  713. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
  714. STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
  715. STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
  716. STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
  717. MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
  718. STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
  719. MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
  720. STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
  721. . = 0x4c00
  722. .globl system_call_relon_pSeries
  723. system_call_relon_pSeries:
  724. HMT_MEDIUM
  725. SYSCALL_PSERIES_1
  726. SYSCALL_PSERIES_2_DIRECT
  727. SYSCALL_PSERIES_3
  728. STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
  729. . = 0x4e00
  730. b . /* Can't happen, see v2.07 Book III-S section 6.5 */
  731. . = 0x4e20
  732. b . /* Can't happen, see v2.07 Book III-S section 6.5 */
  733. . = 0x4e40
  734. SET_SCRATCH0(r13)
  735. EXCEPTION_PROLOG_0(PACA_EXGEN)
  736. b emulation_assist_relon_hv
  737. . = 0x4e60
  738. b . /* Can't happen, see v2.07 Book III-S section 6.5 */
  739. . = 0x4e80
  740. SET_SCRATCH0(r13)
  741. EXCEPTION_PROLOG_0(PACA_EXGEN)
  742. b h_doorbell_relon_hv
  743. performance_monitor_relon_pSeries_1:
  744. . = 0x4f00
  745. SET_SCRATCH0(r13)
  746. EXCEPTION_PROLOG_0(PACA_EXGEN)
  747. b performance_monitor_relon_pSeries
  748. altivec_unavailable_relon_pSeries_1:
  749. . = 0x4f20
  750. SET_SCRATCH0(r13)
  751. EXCEPTION_PROLOG_0(PACA_EXGEN)
  752. b altivec_unavailable_relon_pSeries
  753. vsx_unavailable_relon_pSeries_1:
  754. . = 0x4f40
  755. SET_SCRATCH0(r13)
  756. EXCEPTION_PROLOG_0(PACA_EXGEN)
  757. b vsx_unavailable_relon_pSeries
  758. facility_unavailable_relon_trampoline:
  759. . = 0x4f60
  760. SET_SCRATCH0(r13)
  761. EXCEPTION_PROLOG_0(PACA_EXGEN)
  762. b facility_unavailable_relon_pSeries
  763. hv_facility_unavailable_relon_trampoline:
  764. . = 0x4f80
  765. SET_SCRATCH0(r13)
  766. EXCEPTION_PROLOG_0(PACA_EXGEN)
  767. b facility_unavailable_relon_hv
  768. STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
  769. #ifdef CONFIG_PPC_DENORMALISATION
  770. . = 0x5500
  771. b denorm_exception_hv
  772. #endif
  773. STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
  774. /* Other future vectors */
  775. .align 7
  776. .globl __end_interrupts
  777. __end_interrupts:
  778. .align 7
  779. system_call_entry_direct:
  780. #if defined(CONFIG_RELOCATABLE)
  781. /* The first level prologue may have used LR to get here, saving
  782. * orig in r10. To save hacking/ifdeffing common code, restore here.
  783. */
  784. mtlr r10
  785. #endif
  786. system_call_entry:
  787. b system_call_common
  788. ppc64_runlatch_on_trampoline:
  789. b .__ppc64_runlatch_on
  790. /*
  791. * Here we have detected that the kernel stack pointer is bad.
  792. * R9 contains the saved CR, r13 points to the paca,
  793. * r10 contains the (bad) kernel stack pointer,
  794. * r11 and r12 contain the saved SRR0 and SRR1.
  795. * We switch to using an emergency stack, save the registers there,
  796. * and call kernel_bad_stack(), which panics.
  797. */
  798. bad_stack:
  799. ld r1,PACAEMERGSP(r13)
  800. subi r1,r1,64+INT_FRAME_SIZE
  801. std r9,_CCR(r1)
  802. std r10,GPR1(r1)
  803. std r11,_NIP(r1)
  804. std r12,_MSR(r1)
  805. mfspr r11,SPRN_DAR
  806. mfspr r12,SPRN_DSISR
  807. std r11,_DAR(r1)
  808. std r12,_DSISR(r1)
  809. mflr r10
  810. mfctr r11
  811. mfxer r12
  812. std r10,_LINK(r1)
  813. std r11,_CTR(r1)
  814. std r12,_XER(r1)
  815. SAVE_GPR(0,r1)
  816. SAVE_GPR(2,r1)
  817. ld r10,EX_R3(r3)
  818. std r10,GPR3(r1)
  819. SAVE_GPR(4,r1)
  820. SAVE_4GPRS(5,r1)
  821. ld r9,EX_R9(r3)
  822. ld r10,EX_R10(r3)
  823. SAVE_2GPRS(9,r1)
  824. ld r9,EX_R11(r3)
  825. ld r10,EX_R12(r3)
  826. ld r11,EX_R13(r3)
  827. std r9,GPR11(r1)
  828. std r10,GPR12(r1)
  829. std r11,GPR13(r1)
  830. BEGIN_FTR_SECTION
  831. ld r10,EX_CFAR(r3)
  832. std r10,ORIG_GPR3(r1)
  833. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  834. SAVE_8GPRS(14,r1)
  835. SAVE_10GPRS(22,r1)
  836. lhz r12,PACA_TRAP_SAVE(r13)
  837. std r12,_TRAP(r1)
  838. addi r11,r1,INT_FRAME_SIZE
  839. std r11,0(r1)
  840. li r12,0
  841. std r12,0(r11)
  842. ld r2,PACATOC(r13)
  843. ld r11,exception_marker@toc(r2)
  844. std r12,RESULT(r1)
  845. std r11,STACK_FRAME_OVERHEAD-16(r1)
  846. 1: addi r3,r1,STACK_FRAME_OVERHEAD
  847. bl .kernel_bad_stack
  848. b 1b
  849. /*
  850. * Here r13 points to the paca, r9 contains the saved CR,
  851. * SRR0 and SRR1 are saved in r11 and r12,
  852. * r9 - r13 are saved in paca->exgen.
  853. */
  854. .align 7
  855. .globl data_access_common
  856. data_access_common:
  857. mfspr r10,SPRN_DAR
  858. std r10,PACA_EXGEN+EX_DAR(r13)
  859. mfspr r10,SPRN_DSISR
  860. stw r10,PACA_EXGEN+EX_DSISR(r13)
  861. EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
  862. DISABLE_INTS
  863. ld r12,_MSR(r1)
  864. ld r3,PACA_EXGEN+EX_DAR(r13)
  865. lwz r4,PACA_EXGEN+EX_DSISR(r13)
  866. li r5,0x300
  867. b .do_hash_page /* Try to handle as hpte fault */
  868. .align 7
  869. .globl h_data_storage_common
  870. h_data_storage_common:
  871. mfspr r10,SPRN_HDAR
  872. std r10,PACA_EXGEN+EX_DAR(r13)
  873. mfspr r10,SPRN_HDSISR
  874. stw r10,PACA_EXGEN+EX_DSISR(r13)
  875. EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
  876. bl .save_nvgprs
  877. DISABLE_INTS
  878. addi r3,r1,STACK_FRAME_OVERHEAD
  879. bl .unknown_exception
  880. b .ret_from_except
  881. .align 7
  882. .globl instruction_access_common
  883. instruction_access_common:
  884. EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
  885. DISABLE_INTS
  886. ld r12,_MSR(r1)
  887. ld r3,_NIP(r1)
  888. andis. r4,r12,0x5820
  889. li r5,0x400
  890. b .do_hash_page /* Try to handle as hpte fault */
  891. STD_EXCEPTION_COMMON(0xe20, h_instr_storage, .unknown_exception)
  892. /*
  893. * Here is the common SLB miss user that is used when going to virtual
  894. * mode for SLB misses, that is currently not used
  895. */
  896. #ifdef __DISABLED__
  897. .align 7
  898. .globl slb_miss_user_common
  899. slb_miss_user_common:
  900. mflr r10
  901. std r3,PACA_EXGEN+EX_DAR(r13)
  902. stw r9,PACA_EXGEN+EX_CCR(r13)
  903. std r10,PACA_EXGEN+EX_LR(r13)
  904. std r11,PACA_EXGEN+EX_SRR0(r13)
  905. bl .slb_allocate_user
  906. ld r10,PACA_EXGEN+EX_LR(r13)
  907. ld r3,PACA_EXGEN+EX_R3(r13)
  908. lwz r9,PACA_EXGEN+EX_CCR(r13)
  909. ld r11,PACA_EXGEN+EX_SRR0(r13)
  910. mtlr r10
  911. beq- slb_miss_fault
  912. andi. r10,r12,MSR_RI /* check for unrecoverable exception */
  913. beq- unrecov_user_slb
  914. mfmsr r10
  915. .machine push
  916. .machine "power4"
  917. mtcrf 0x80,r9
  918. .machine pop
  919. clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
  920. mtmsrd r10,1
  921. mtspr SRR0,r11
  922. mtspr SRR1,r12
  923. ld r9,PACA_EXGEN+EX_R9(r13)
  924. ld r10,PACA_EXGEN+EX_R10(r13)
  925. ld r11,PACA_EXGEN+EX_R11(r13)
  926. ld r12,PACA_EXGEN+EX_R12(r13)
  927. ld r13,PACA_EXGEN+EX_R13(r13)
  928. rfid
  929. b .
  930. slb_miss_fault:
  931. EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
  932. ld r4,PACA_EXGEN+EX_DAR(r13)
  933. li r5,0
  934. std r4,_DAR(r1)
  935. std r5,_DSISR(r1)
  936. b handle_page_fault
  937. unrecov_user_slb:
  938. EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
  939. DISABLE_INTS
  940. bl .save_nvgprs
  941. 1: addi r3,r1,STACK_FRAME_OVERHEAD
  942. bl .unrecoverable_exception
  943. b 1b
  944. #endif /* __DISABLED__ */
  945. .align 7
  946. .globl alignment_common
  947. alignment_common:
  948. mfspr r10,SPRN_DAR
  949. std r10,PACA_EXGEN+EX_DAR(r13)
  950. mfspr r10,SPRN_DSISR
  951. stw r10,PACA_EXGEN+EX_DSISR(r13)
  952. EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
  953. ld r3,PACA_EXGEN+EX_DAR(r13)
  954. lwz r4,PACA_EXGEN+EX_DSISR(r13)
  955. std r3,_DAR(r1)
  956. std r4,_DSISR(r1)
  957. bl .save_nvgprs
  958. DISABLE_INTS
  959. addi r3,r1,STACK_FRAME_OVERHEAD
  960. bl .alignment_exception
  961. b .ret_from_except
  962. .align 7
  963. .globl program_check_common
  964. program_check_common:
  965. EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
  966. bl .save_nvgprs
  967. DISABLE_INTS
  968. addi r3,r1,STACK_FRAME_OVERHEAD
  969. bl .program_check_exception
  970. b .ret_from_except
  971. .align 7
  972. .globl fp_unavailable_common
  973. fp_unavailable_common:
  974. EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
  975. bne 1f /* if from user, just load it up */
  976. bl .save_nvgprs
  977. DISABLE_INTS
  978. addi r3,r1,STACK_FRAME_OVERHEAD
  979. bl .kernel_fp_unavailable_exception
  980. BUG_OPCODE
  981. 1:
  982. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  983. BEGIN_FTR_SECTION
  984. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  985. * transaction), go do TM stuff
  986. */
  987. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  988. bne- 2f
  989. END_FTR_SECTION_IFSET(CPU_FTR_TM)
  990. #endif
  991. bl .load_up_fpu
  992. b fast_exception_return
  993. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  994. 2: /* User process was in a transaction */
  995. bl .save_nvgprs
  996. DISABLE_INTS
  997. addi r3,r1,STACK_FRAME_OVERHEAD
  998. bl .fp_unavailable_tm
  999. b .ret_from_except
  1000. #endif
  1001. .align 7
  1002. .globl altivec_unavailable_common
  1003. altivec_unavailable_common:
  1004. EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
  1005. #ifdef CONFIG_ALTIVEC
  1006. BEGIN_FTR_SECTION
  1007. beq 1f
  1008. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1009. BEGIN_FTR_SECTION_NESTED(69)
  1010. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  1011. * transaction), go do TM stuff
  1012. */
  1013. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  1014. bne- 2f
  1015. END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
  1016. #endif
  1017. bl .load_up_altivec
  1018. b fast_exception_return
  1019. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1020. 2: /* User process was in a transaction */
  1021. bl .save_nvgprs
  1022. DISABLE_INTS
  1023. addi r3,r1,STACK_FRAME_OVERHEAD
  1024. bl .altivec_unavailable_tm
  1025. b .ret_from_except
  1026. #endif
  1027. 1:
  1028. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  1029. #endif
  1030. bl .save_nvgprs
  1031. DISABLE_INTS
  1032. addi r3,r1,STACK_FRAME_OVERHEAD
  1033. bl .altivec_unavailable_exception
  1034. b .ret_from_except
  1035. .align 7
  1036. .globl vsx_unavailable_common
  1037. vsx_unavailable_common:
  1038. EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
  1039. #ifdef CONFIG_VSX
  1040. BEGIN_FTR_SECTION
  1041. beq 1f
  1042. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1043. BEGIN_FTR_SECTION_NESTED(69)
  1044. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  1045. * transaction), go do TM stuff
  1046. */
  1047. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  1048. bne- 2f
  1049. END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
  1050. #endif
  1051. b .load_up_vsx
  1052. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1053. 2: /* User process was in a transaction */
  1054. bl .save_nvgprs
  1055. DISABLE_INTS
  1056. addi r3,r1,STACK_FRAME_OVERHEAD
  1057. bl .vsx_unavailable_tm
  1058. b .ret_from_except
  1059. #endif
  1060. 1:
  1061. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  1062. #endif
  1063. bl .save_nvgprs
  1064. DISABLE_INTS
  1065. addi r3,r1,STACK_FRAME_OVERHEAD
  1066. bl .vsx_unavailable_exception
  1067. b .ret_from_except
  1068. STD_EXCEPTION_COMMON(0xf60, facility_unavailable, .facility_unavailable_exception)
  1069. .align 7
  1070. .globl __end_handlers
  1071. __end_handlers:
  1072. /* Equivalents to the above handlers for relocation-on interrupt vectors */
  1073. STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
  1074. MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
  1075. STD_RELON_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
  1076. STD_RELON_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
  1077. STD_RELON_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
  1078. STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
  1079. STD_RELON_EXCEPTION_HV_OOL(0xf80, facility_unavailable)
  1080. #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
  1081. /*
  1082. * Data area reserved for FWNMI option.
  1083. * This address (0x7000) is fixed by the RPA.
  1084. */
  1085. .= 0x7000
  1086. .globl fwnmi_data_area
  1087. fwnmi_data_area:
  1088. /* pseries and powernv need to keep the whole page from
  1089. * 0x7000 to 0x8000 free for use by the firmware
  1090. */
  1091. . = 0x8000
  1092. #endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
  1093. /* Space for CPU0's segment table */
  1094. .balign 4096
  1095. .globl initial_stab
  1096. initial_stab:
  1097. .space 4096
  1098. #ifdef CONFIG_PPC_POWERNV
  1099. _GLOBAL(opal_mc_secondary_handler)
  1100. HMT_MEDIUM_PPR_DISCARD
  1101. SET_SCRATCH0(r13)
  1102. GET_PACA(r13)
  1103. clrldi r3,r3,2
  1104. tovirt(r3,r3)
  1105. std r3,PACA_OPAL_MC_EVT(r13)
  1106. ld r13,OPAL_MC_SRR0(r3)
  1107. mtspr SPRN_SRR0,r13
  1108. ld r13,OPAL_MC_SRR1(r3)
  1109. mtspr SPRN_SRR1,r13
  1110. ld r3,OPAL_MC_GPR3(r3)
  1111. GET_SCRATCH0(r13)
  1112. b machine_check_pSeries
  1113. #endif /* CONFIG_PPC_POWERNV */
  1114. /*
  1115. * r13 points to the PACA, r9 contains the saved CR,
  1116. * r12 contain the saved SRR1, SRR0 is still ready for return
  1117. * r3 has the faulting address
  1118. * r9 - r13 are saved in paca->exslb.
  1119. * r3 is saved in paca->slb_r3
  1120. * We assume we aren't going to take any exceptions during this procedure.
  1121. */
  1122. _GLOBAL(slb_miss_realmode)
  1123. mflr r10
  1124. #ifdef CONFIG_RELOCATABLE
  1125. mtctr r11
  1126. #endif
  1127. stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
  1128. std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
  1129. bl .slb_allocate_realmode
  1130. /* All done -- return from exception. */
  1131. ld r10,PACA_EXSLB+EX_LR(r13)
  1132. ld r3,PACA_EXSLB+EX_R3(r13)
  1133. lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
  1134. mtlr r10
  1135. andi. r10,r12,MSR_RI /* check for unrecoverable exception */
  1136. beq- 2f
  1137. .machine push
  1138. .machine "power4"
  1139. mtcrf 0x80,r9
  1140. mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
  1141. .machine pop
  1142. RESTORE_PPR_PACA(PACA_EXSLB, r9)
  1143. ld r9,PACA_EXSLB+EX_R9(r13)
  1144. ld r10,PACA_EXSLB+EX_R10(r13)
  1145. ld r11,PACA_EXSLB+EX_R11(r13)
  1146. ld r12,PACA_EXSLB+EX_R12(r13)
  1147. ld r13,PACA_EXSLB+EX_R13(r13)
  1148. rfid
  1149. b . /* prevent speculative execution */
  1150. 2: mfspr r11,SPRN_SRR0
  1151. ld r10,PACAKBASE(r13)
  1152. LOAD_HANDLER(r10,unrecov_slb)
  1153. mtspr SPRN_SRR0,r10
  1154. ld r10,PACAKMSR(r13)
  1155. mtspr SPRN_SRR1,r10
  1156. rfid
  1157. b .
  1158. unrecov_slb:
  1159. EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
  1160. DISABLE_INTS
  1161. bl .save_nvgprs
  1162. 1: addi r3,r1,STACK_FRAME_OVERHEAD
  1163. bl .unrecoverable_exception
  1164. b 1b
  1165. #ifdef CONFIG_PPC_970_NAP
  1166. power4_fixup_nap:
  1167. andc r9,r9,r10
  1168. std r9,TI_LOCAL_FLAGS(r11)
  1169. ld r10,_LINK(r1) /* make idle task do the */
  1170. std r10,_NIP(r1) /* equivalent of a blr */
  1171. blr
  1172. #endif
  1173. /*
  1174. * Hash table stuff
  1175. */
  1176. .align 7
  1177. _STATIC(do_hash_page)
  1178. std r3,_DAR(r1)
  1179. std r4,_DSISR(r1)
  1180. andis. r0,r4,0xa410 /* weird error? */
  1181. bne- handle_page_fault /* if not, try to insert a HPTE */
  1182. andis. r0,r4,DSISR_DABRMATCH@h
  1183. bne- handle_dabr_fault
  1184. BEGIN_FTR_SECTION
  1185. andis. r0,r4,0x0020 /* Is it a segment table fault? */
  1186. bne- do_ste_alloc /* If so handle it */
  1187. END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
  1188. CURRENT_THREAD_INFO(r11, r1)
  1189. lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */
  1190. andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */
  1191. bne 77f /* then don't call hash_page now */
  1192. /*
  1193. * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
  1194. * accessing a userspace segment (even from the kernel). We assume
  1195. * kernel addresses always have the high bit set.
  1196. */
  1197. rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
  1198. rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
  1199. orc r0,r12,r0 /* MSR_PR | ~high_bit */
  1200. rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
  1201. ori r4,r4,1 /* add _PAGE_PRESENT */
  1202. rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
  1203. /*
  1204. * r3 contains the faulting address
  1205. * r4 contains the required access permissions
  1206. * r5 contains the trap number
  1207. *
  1208. * at return r3 = 0 for success, 1 for page fault, negative for error
  1209. */
  1210. bl .hash_page /* build HPTE if possible */
  1211. cmpdi r3,0 /* see if hash_page succeeded */
  1212. /* Success */
  1213. beq fast_exc_return_irq /* Return from exception on success */
  1214. /* Error */
  1215. blt- 13f
  1216. /* Here we have a page fault that hash_page can't handle. */
  1217. handle_page_fault:
  1218. 11: ld r4,_DAR(r1)
  1219. ld r5,_DSISR(r1)
  1220. addi r3,r1,STACK_FRAME_OVERHEAD
  1221. bl .do_page_fault
  1222. cmpdi r3,0
  1223. beq+ 12f
  1224. bl .save_nvgprs
  1225. mr r5,r3
  1226. addi r3,r1,STACK_FRAME_OVERHEAD
  1227. lwz r4,_DAR(r1)
  1228. bl .bad_page_fault
  1229. b .ret_from_except
  1230. /* We have a data breakpoint exception - handle it */
  1231. handle_dabr_fault:
  1232. bl .save_nvgprs
  1233. ld r4,_DAR(r1)
  1234. ld r5,_DSISR(r1)
  1235. addi r3,r1,STACK_FRAME_OVERHEAD
  1236. bl .do_break
  1237. 12: b .ret_from_except_lite
  1238. /* We have a page fault that hash_page could handle but HV refused
  1239. * the PTE insertion
  1240. */
  1241. 13: bl .save_nvgprs
  1242. mr r5,r3
  1243. addi r3,r1,STACK_FRAME_OVERHEAD
  1244. ld r4,_DAR(r1)
  1245. bl .low_hash_fault
  1246. b .ret_from_except
  1247. /*
  1248. * We come here as a result of a DSI at a point where we don't want
  1249. * to call hash_page, such as when we are accessing memory (possibly
  1250. * user memory) inside a PMU interrupt that occurred while interrupts
  1251. * were soft-disabled. We want to invoke the exception handler for
  1252. * the access, or panic if there isn't a handler.
  1253. */
  1254. 77: bl .save_nvgprs
  1255. mr r4,r3
  1256. addi r3,r1,STACK_FRAME_OVERHEAD
  1257. li r5,SIGSEGV
  1258. bl .bad_page_fault
  1259. b .ret_from_except
  1260. /* here we have a segment miss */
  1261. do_ste_alloc:
  1262. bl .ste_allocate /* try to insert stab entry */
  1263. cmpdi r3,0
  1264. bne- handle_page_fault
  1265. b fast_exception_return
  1266. /*
  1267. * r13 points to the PACA, r9 contains the saved CR,
  1268. * r11 and r12 contain the saved SRR0 and SRR1.
  1269. * r9 - r13 are saved in paca->exslb.
  1270. * We assume we aren't going to take any exceptions during this procedure.
  1271. * We assume (DAR >> 60) == 0xc.
  1272. */
  1273. .align 7
  1274. _GLOBAL(do_stab_bolted)
  1275. stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
  1276. std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
  1277. mfspr r11,SPRN_DAR /* ea */
  1278. /*
  1279. * check for bad kernel/user address
  1280. * (ea & ~REGION_MASK) >= PGTABLE_RANGE
  1281. */
  1282. rldicr. r9,r11,4,(63 - 46 - 4)
  1283. li r9,0 /* VSID = 0 for bad address */
  1284. bne- 0f
  1285. /*
  1286. * Calculate VSID:
  1287. * This is the kernel vsid, we take the top for context from
  1288. * the range. context = (MAX_USER_CONTEXT) + ((ea >> 60) - 0xc) + 1
  1289. * Here we know that (ea >> 60) == 0xc
  1290. */
  1291. lis r9,(MAX_USER_CONTEXT + 1)@ha
  1292. addi r9,r9,(MAX_USER_CONTEXT + 1)@l
  1293. srdi r10,r11,SID_SHIFT
  1294. rldimi r10,r9,ESID_BITS,0 /* proto vsid */
  1295. ASM_VSID_SCRAMBLE(r10, r9, 256M)
  1296. rldic r9,r10,12,16 /* r9 = vsid << 12 */
  1297. 0:
  1298. /* Hash to the primary group */
  1299. ld r10,PACASTABVIRT(r13)
  1300. srdi r11,r11,SID_SHIFT
  1301. rldimi r10,r11,7,52 /* r10 = first ste of the group */
  1302. /* Search the primary group for a free entry */
  1303. 1: ld r11,0(r10) /* Test valid bit of the current ste */
  1304. andi. r11,r11,0x80
  1305. beq 2f
  1306. addi r10,r10,16
  1307. andi. r11,r10,0x70
  1308. bne 1b
  1309. /* Stick for only searching the primary group for now. */
  1310. /* At least for now, we use a very simple random castout scheme */
  1311. /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
  1312. mftb r11
  1313. rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
  1314. ori r11,r11,0x10
  1315. /* r10 currently points to an ste one past the group of interest */
  1316. /* make it point to the randomly selected entry */
  1317. subi r10,r10,128
  1318. or r10,r10,r11 /* r10 is the entry to invalidate */
  1319. isync /* mark the entry invalid */
  1320. ld r11,0(r10)
  1321. rldicl r11,r11,56,1 /* clear the valid bit */
  1322. rotldi r11,r11,8
  1323. std r11,0(r10)
  1324. sync
  1325. clrrdi r11,r11,28 /* Get the esid part of the ste */
  1326. slbie r11
  1327. 2: std r9,8(r10) /* Store the vsid part of the ste */
  1328. eieio
  1329. mfspr r11,SPRN_DAR /* Get the new esid */
  1330. clrrdi r11,r11,28 /* Permits a full 32b of ESID */
  1331. ori r11,r11,0x90 /* Turn on valid and kp */
  1332. std r11,0(r10) /* Put new entry back into the stab */
  1333. sync
  1334. /* All done -- return from exception. */
  1335. lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
  1336. ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
  1337. andi. r10,r12,MSR_RI
  1338. beq- unrecov_slb
  1339. mtcrf 0x80,r9 /* restore CR */
  1340. mfmsr r10
  1341. clrrdi r10,r10,2
  1342. mtmsrd r10,1
  1343. mtspr SPRN_SRR0,r11
  1344. mtspr SPRN_SRR1,r12
  1345. ld r9,PACA_EXSLB+EX_R9(r13)
  1346. ld r10,PACA_EXSLB+EX_R10(r13)
  1347. ld r11,PACA_EXSLB+EX_R11(r13)
  1348. ld r12,PACA_EXSLB+EX_R12(r13)
  1349. ld r13,PACA_EXSLB+EX_R13(r13)
  1350. rfid
  1351. b . /* prevent speculative execution */