hash_low_64.S 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. /*
  2. * ppc64 MMU hashtable management routines
  3. *
  4. * (c) Copyright IBM Corp. 2003, 2005
  5. *
  6. * Maintained by: Benjamin Herrenschmidt
  7. * <benh@kernel.crashing.org>
  8. *
  9. * This file is covered by the GNU Public Licence v2 as
  10. * described in the kernel's COPYING file.
  11. */
  12. #include <asm/reg.h>
  13. #include <asm/pgtable.h>
  14. #include <asm/mmu.h>
  15. #include <asm/page.h>
  16. #include <asm/types.h>
  17. #include <asm/ppc_asm.h>
  18. #include <asm/asm-offsets.h>
  19. #include <asm/cputable.h>
  20. .text
  21. /*
  22. * Stackframe:
  23. *
  24. * +-> Back chain (SP + 256)
  25. * | General register save area (SP + 112)
  26. * | Parameter save area (SP + 48)
  27. * | TOC save area (SP + 40)
  28. * | link editor doubleword (SP + 32)
  29. * | compiler doubleword (SP + 24)
  30. * | LR save area (SP + 16)
  31. * | CR save area (SP + 8)
  32. * SP ---> +-- Back chain (SP + 0)
  33. */
  34. #define STACKFRAMESIZE 256
  35. /* Save parameters offsets */
  36. #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
  37. /* Save non-volatile offsets */
  38. #define STK_REG(i) (112 + ((i)-14)*8)
  39. #ifndef CONFIG_PPC_64K_PAGES
  40. /*****************************************************************************
  41. * *
  42. * 4K SW & 4K HW pages implementation *
  43. * *
  44. *****************************************************************************/
  45. /*
  46. * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  47. * pte_t *ptep, unsigned long trap, int local, int ssize)
  48. *
  49. * Adds a 4K page to the hash table in a segment of 4K pages only
  50. */
  51. _GLOBAL(__hash_page_4K)
  52. mflr r0
  53. std r0,16(r1)
  54. stdu r1,-STACKFRAMESIZE(r1)
  55. /* Save all params that we need after a function call */
  56. std r6,STK_PARM(r6)(r1)
  57. std r8,STK_PARM(r8)(r1)
  58. std r9,STK_PARM(r9)(r1)
  59. /* Add _PAGE_PRESENT to access */
  60. ori r4,r4,_PAGE_PRESENT
  61. /* Save non-volatile registers.
  62. * r31 will hold "old PTE"
  63. * r30 is "new PTE"
  64. * r29 is "va"
  65. * r28 is a hash value
  66. * r27 is hashtab mask (maybe dynamic patched instead ?)
  67. */
  68. std r27,STK_REG(r27)(r1)
  69. std r28,STK_REG(r28)(r1)
  70. std r29,STK_REG(r29)(r1)
  71. std r30,STK_REG(r30)(r1)
  72. std r31,STK_REG(r31)(r1)
  73. /* Step 1:
  74. *
  75. * Check permissions, atomically mark the linux PTE busy
  76. * and hashed.
  77. */
  78. 1:
  79. ldarx r31,0,r6
  80. /* Check access rights (access & ~(pte_val(*ptep))) */
  81. andc. r0,r4,r31
  82. bne- htab_wrong_access
  83. /* Check if PTE is busy */
  84. andi. r0,r31,_PAGE_BUSY
  85. /* If so, just bail out and refault if needed. Someone else
  86. * is changing this PTE anyway and might hash it.
  87. */
  88. bne- htab_bail_ok
  89. /* Prepare new PTE value (turn access RW into DIRTY, then
  90. * add BUSY,HASHPTE and ACCESSED)
  91. */
  92. rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  93. or r30,r30,r31
  94. ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
  95. /* Write the linux PTE atomically (setting busy) */
  96. stdcx. r30,0,r6
  97. bne- 1b
  98. isync
  99. /* Step 2:
  100. *
  101. * Insert/Update the HPTE in the hash table. At this point,
  102. * r4 (access) is re-useable, we use it for the new HPTE flags
  103. */
  104. BEGIN_FTR_SECTION
  105. cmpdi r9,0 /* check segment size */
  106. bne 3f
  107. END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
  108. /* Calc va and put it in r29 */
  109. rldicr r29,r5,28,63-28
  110. rldicl r3,r3,0,36
  111. or r29,r3,r29
  112. /* Calculate hash value for primary slot and store it in r28 */
  113. rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
  114. rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
  115. xor r28,r5,r0
  116. b 4f
  117. 3: /* Calc VA and hash in r29 and r28 for 1T segment */
  118. sldi r29,r5,40 /* vsid << 40 */
  119. clrldi r3,r3,24 /* ea & 0xffffffffff */
  120. rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
  121. clrldi r5,r5,40 /* vsid & 0xffffff */
  122. rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
  123. xor r28,r28,r5
  124. or r29,r3,r29 /* VA */
  125. xor r28,r28,r0 /* hash */
  126. /* Convert linux PTE bits into HW equivalents */
  127. 4: andi. r3,r30,0x1fe /* Get basic set of flags */
  128. xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
  129. rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
  130. rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
  131. and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
  132. andc r0,r30,r0 /* r0 = pte & ~r0 */
  133. rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
  134. ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
  135. /* We eventually do the icache sync here (maybe inline that
  136. * code rather than call a C function...)
  137. */
  138. BEGIN_FTR_SECTION
  139. mr r4,r30
  140. mr r5,r7
  141. bl .hash_page_do_lazy_icache
  142. END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
  143. /* At this point, r3 contains new PP bits, save them in
  144. * place of "access" in the param area (sic)
  145. */
  146. std r3,STK_PARM(r4)(r1)
  147. /* Get htab_hash_mask */
  148. ld r4,htab_hash_mask@got(2)
  149. ld r27,0(r4) /* htab_hash_mask -> r27 */
  150. /* Check if we may already be in the hashtable, in this case, we
  151. * go to out-of-line code to try to modify the HPTE
  152. */
  153. andi. r0,r31,_PAGE_HASHPTE
  154. bne htab_modify_pte
  155. htab_insert_pte:
  156. /* Clear hpte bits in new pte (we also clear BUSY btw) and
  157. * add _PAGE_HASHPTE
  158. */
  159. lis r0,_PAGE_HPTEFLAGS@h
  160. ori r0,r0,_PAGE_HPTEFLAGS@l
  161. andc r30,r30,r0
  162. ori r30,r30,_PAGE_HASHPTE
  163. /* physical address r5 */
  164. rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
  165. sldi r5,r5,PAGE_SHIFT
  166. /* Calculate primary group hash */
  167. and r0,r28,r27
  168. rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
  169. /* Call ppc_md.hpte_insert */
  170. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  171. mr r4,r29 /* Retreive va */
  172. li r7,0 /* !bolted, !secondary */
  173. li r8,MMU_PAGE_4K /* page size */
  174. ld r9,STK_PARM(r9)(r1) /* segment size */
  175. _GLOBAL(htab_call_hpte_insert1)
  176. bl . /* Patched by htab_finish_init() */
  177. cmpdi 0,r3,0
  178. bge htab_pte_insert_ok /* Insertion successful */
  179. cmpdi 0,r3,-2 /* Critical failure */
  180. beq- htab_pte_insert_failure
  181. /* Now try secondary slot */
  182. /* physical address r5 */
  183. rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
  184. sldi r5,r5,PAGE_SHIFT
  185. /* Calculate secondary group hash */
  186. andc r0,r27,r28
  187. rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
  188. /* Call ppc_md.hpte_insert */
  189. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  190. mr r4,r29 /* Retreive va */
  191. li r7,HPTE_V_SECONDARY /* !bolted, secondary */
  192. li r8,MMU_PAGE_4K /* page size */
  193. ld r9,STK_PARM(r9)(r1) /* segment size */
  194. _GLOBAL(htab_call_hpte_insert2)
  195. bl . /* Patched by htab_finish_init() */
  196. cmpdi 0,r3,0
  197. bge+ htab_pte_insert_ok /* Insertion successful */
  198. cmpdi 0,r3,-2 /* Critical failure */
  199. beq- htab_pte_insert_failure
  200. /* Both are full, we need to evict something */
  201. mftb r0
  202. /* Pick a random group based on TB */
  203. andi. r0,r0,1
  204. mr r5,r28
  205. bne 2f
  206. not r5,r5
  207. 2: and r0,r5,r27
  208. rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  209. /* Call ppc_md.hpte_remove */
  210. _GLOBAL(htab_call_hpte_remove)
  211. bl . /* Patched by htab_finish_init() */
  212. /* Try all again */
  213. b htab_insert_pte
  214. htab_bail_ok:
  215. li r3,0
  216. b htab_bail
  217. htab_pte_insert_ok:
  218. /* Insert slot number & secondary bit in PTE */
  219. rldimi r30,r3,12,63-15
  220. /* Write out the PTE with a normal write
  221. * (maybe add eieio may be good still ?)
  222. */
  223. htab_write_out_pte:
  224. ld r6,STK_PARM(r6)(r1)
  225. std r30,0(r6)
  226. li r3, 0
  227. htab_bail:
  228. ld r27,STK_REG(r27)(r1)
  229. ld r28,STK_REG(r28)(r1)
  230. ld r29,STK_REG(r29)(r1)
  231. ld r30,STK_REG(r30)(r1)
  232. ld r31,STK_REG(r31)(r1)
  233. addi r1,r1,STACKFRAMESIZE
  234. ld r0,16(r1)
  235. mtlr r0
  236. blr
  237. htab_modify_pte:
  238. /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
  239. mr r4,r3
  240. rlwinm r3,r31,32-12,29,31
  241. /* Secondary group ? if yes, get a inverted hash value */
  242. mr r5,r28
  243. andi. r0,r31,_PAGE_SECONDARY
  244. beq 1f
  245. not r5,r5
  246. 1:
  247. /* Calculate proper slot value for ppc_md.hpte_updatepp */
  248. and r0,r5,r27
  249. rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  250. add r3,r0,r3 /* add slot idx */
  251. /* Call ppc_md.hpte_updatepp */
  252. mr r5,r29 /* va */
  253. li r6,MMU_PAGE_4K /* page size */
  254. ld r7,STK_PARM(r9)(r1) /* segment size */
  255. ld r8,STK_PARM(r8)(r1) /* get "local" param */
  256. _GLOBAL(htab_call_hpte_updatepp)
  257. bl . /* Patched by htab_finish_init() */
  258. /* if we failed because typically the HPTE wasn't really here
  259. * we try an insertion.
  260. */
  261. cmpdi 0,r3,-1
  262. beq- htab_insert_pte
  263. /* Clear the BUSY bit and Write out the PTE */
  264. li r0,_PAGE_BUSY
  265. andc r30,r30,r0
  266. b htab_write_out_pte
  267. htab_wrong_access:
  268. /* Bail out clearing reservation */
  269. stdcx. r31,0,r6
  270. li r3,1
  271. b htab_bail
  272. htab_pte_insert_failure:
  273. /* Bail out restoring old PTE */
  274. ld r6,STK_PARM(r6)(r1)
  275. std r31,0(r6)
  276. li r3,-1
  277. b htab_bail
  278. #else /* CONFIG_PPC_64K_PAGES */
  279. /*****************************************************************************
  280. * *
  281. * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
  282. * *
  283. *****************************************************************************/
  284. /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  285. * pte_t *ptep, unsigned long trap, int local, int ssize,
  286. * int subpg_prot)
  287. */
  288. /*
  289. * For now, we do NOT implement Admixed pages
  290. */
  291. _GLOBAL(__hash_page_4K)
  292. mflr r0
  293. std r0,16(r1)
  294. stdu r1,-STACKFRAMESIZE(r1)
  295. /* Save all params that we need after a function call */
  296. std r6,STK_PARM(r6)(r1)
  297. std r8,STK_PARM(r8)(r1)
  298. std r9,STK_PARM(r9)(r1)
  299. /* Add _PAGE_PRESENT to access */
  300. ori r4,r4,_PAGE_PRESENT
  301. /* Save non-volatile registers.
  302. * r31 will hold "old PTE"
  303. * r30 is "new PTE"
  304. * r29 is "va"
  305. * r28 is a hash value
  306. * r27 is hashtab mask (maybe dynamic patched instead ?)
  307. * r26 is the hidx mask
  308. * r25 is the index in combo page
  309. */
  310. std r25,STK_REG(r25)(r1)
  311. std r26,STK_REG(r26)(r1)
  312. std r27,STK_REG(r27)(r1)
  313. std r28,STK_REG(r28)(r1)
  314. std r29,STK_REG(r29)(r1)
  315. std r30,STK_REG(r30)(r1)
  316. std r31,STK_REG(r31)(r1)
  317. /* Step 1:
  318. *
  319. * Check permissions, atomically mark the linux PTE busy
  320. * and hashed.
  321. */
  322. 1:
  323. ldarx r31,0,r6
  324. /* Check access rights (access & ~(pte_val(*ptep))) */
  325. andc. r0,r4,r31
  326. bne- htab_wrong_access
  327. /* Check if PTE is busy */
  328. andi. r0,r31,_PAGE_BUSY
  329. /* If so, just bail out and refault if needed. Someone else
  330. * is changing this PTE anyway and might hash it.
  331. */
  332. bne- htab_bail_ok
  333. /* Prepare new PTE value (turn access RW into DIRTY, then
  334. * add BUSY and ACCESSED)
  335. */
  336. rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  337. or r30,r30,r31
  338. ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
  339. oris r30,r30,_PAGE_COMBO@h
  340. /* Write the linux PTE atomically (setting busy) */
  341. stdcx. r30,0,r6
  342. bne- 1b
  343. isync
  344. /* Step 2:
  345. *
  346. * Insert/Update the HPTE in the hash table. At this point,
  347. * r4 (access) is re-useable, we use it for the new HPTE flags
  348. */
  349. /* Load the hidx index */
  350. rldicl r25,r3,64-12,60
  351. BEGIN_FTR_SECTION
  352. cmpdi r9,0 /* check segment size */
  353. bne 3f
  354. END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
  355. /* Calc va and put it in r29 */
  356. rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
  357. rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
  358. or r29,r3,r29 /* r29 = va */
  359. /* Calculate hash value for primary slot and store it in r28 */
  360. rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
  361. rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
  362. xor r28,r5,r0
  363. b 4f
  364. 3: /* Calc VA and hash in r29 and r28 for 1T segment */
  365. sldi r29,r5,40 /* vsid << 40 */
  366. clrldi r3,r3,24 /* ea & 0xffffffffff */
  367. rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
  368. clrldi r5,r5,40 /* vsid & 0xffffff */
  369. rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
  370. xor r28,r28,r5
  371. or r29,r3,r29 /* VA */
  372. xor r28,r28,r0 /* hash */
  373. /* Convert linux PTE bits into HW equivalents */
  374. 4:
  375. #ifdef CONFIG_PPC_SUBPAGE_PROT
  376. andc r10,r30,r10
  377. andi. r3,r10,0x1fe /* Get basic set of flags */
  378. rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
  379. #else
  380. andi. r3,r30,0x1fe /* Get basic set of flags */
  381. rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
  382. #endif
  383. xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
  384. rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
  385. and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
  386. andc r0,r3,r0 /* r0 = pte & ~r0 */
  387. rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
  388. ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
  389. /* We eventually do the icache sync here (maybe inline that
  390. * code rather than call a C function...)
  391. */
  392. BEGIN_FTR_SECTION
  393. mr r4,r30
  394. mr r5,r7
  395. bl .hash_page_do_lazy_icache
  396. END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
  397. /* At this point, r3 contains new PP bits, save them in
  398. * place of "access" in the param area (sic)
  399. */
  400. std r3,STK_PARM(r4)(r1)
  401. /* Get htab_hash_mask */
  402. ld r4,htab_hash_mask@got(2)
  403. ld r27,0(r4) /* htab_hash_mask -> r27 */
  404. /* Check if we may already be in the hashtable, in this case, we
  405. * go to out-of-line code to try to modify the HPTE. We look for
  406. * the bit at (1 >> (index + 32))
  407. */
  408. rldicl. r0,r31,64-12,48
  409. li r26,0 /* Default hidx */
  410. beq htab_insert_pte
  411. /*
  412. * Check if the pte was already inserted into the hash table
  413. * as a 64k HW page, and invalidate the 64k HPTE if so.
  414. */
  415. andis. r0,r31,_PAGE_COMBO@h
  416. beq htab_inval_old_hpte
  417. ld r6,STK_PARM(r6)(r1)
  418. ori r26,r6,0x8000 /* Load the hidx mask */
  419. ld r26,0(r26)
  420. addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
  421. rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
  422. bne htab_modify_pte
  423. htab_insert_pte:
  424. /* real page number in r5, PTE RPN value + index */
  425. andis. r0,r31,_PAGE_4K_PFN@h
  426. srdi r5,r31,PTE_RPN_SHIFT
  427. bne- htab_special_pfn
  428. sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
  429. add r5,r5,r25
  430. htab_special_pfn:
  431. sldi r5,r5,HW_PAGE_SHIFT
  432. /* Calculate primary group hash */
  433. and r0,r28,r27
  434. rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  435. /* Call ppc_md.hpte_insert */
  436. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  437. mr r4,r29 /* Retreive va */
  438. li r7,0 /* !bolted, !secondary */
  439. li r8,MMU_PAGE_4K /* page size */
  440. ld r9,STK_PARM(r9)(r1) /* segment size */
  441. _GLOBAL(htab_call_hpte_insert1)
  442. bl . /* patched by htab_finish_init() */
  443. cmpdi 0,r3,0
  444. bge htab_pte_insert_ok /* Insertion successful */
  445. cmpdi 0,r3,-2 /* Critical failure */
  446. beq- htab_pte_insert_failure
  447. /* Now try secondary slot */
  448. /* real page number in r5, PTE RPN value + index */
  449. andis. r0,r31,_PAGE_4K_PFN@h
  450. srdi r5,r31,PTE_RPN_SHIFT
  451. bne- 3f
  452. sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
  453. add r5,r5,r25
  454. 3: sldi r5,r5,HW_PAGE_SHIFT
  455. /* Calculate secondary group hash */
  456. andc r0,r27,r28
  457. rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
  458. /* Call ppc_md.hpte_insert */
  459. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  460. mr r4,r29 /* Retreive va */
  461. li r7,HPTE_V_SECONDARY /* !bolted, secondary */
  462. li r8,MMU_PAGE_4K /* page size */
  463. ld r9,STK_PARM(r9)(r1) /* segment size */
  464. _GLOBAL(htab_call_hpte_insert2)
  465. bl . /* patched by htab_finish_init() */
  466. cmpdi 0,r3,0
  467. bge+ htab_pte_insert_ok /* Insertion successful */
  468. cmpdi 0,r3,-2 /* Critical failure */
  469. beq- htab_pte_insert_failure
  470. /* Both are full, we need to evict something */
  471. mftb r0
  472. /* Pick a random group based on TB */
  473. andi. r0,r0,1
  474. mr r5,r28
  475. bne 2f
  476. not r5,r5
  477. 2: and r0,r5,r27
  478. rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  479. /* Call ppc_md.hpte_remove */
  480. _GLOBAL(htab_call_hpte_remove)
  481. bl . /* patched by htab_finish_init() */
  482. /* Try all again */
  483. b htab_insert_pte
  484. /*
  485. * Call out to C code to invalidate an 64k HW HPTE that is
  486. * useless now that the segment has been switched to 4k pages.
  487. */
  488. htab_inval_old_hpte:
  489. mr r3,r29 /* virtual addr */
  490. mr r4,r31 /* PTE.pte */
  491. li r5,0 /* PTE.hidx */
  492. li r6,MMU_PAGE_64K /* psize */
  493. ld r7,STK_PARM(r9)(r1) /* ssize */
  494. ld r8,STK_PARM(r8)(r1) /* local */
  495. bl .flush_hash_page
  496. /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
  497. lis r0,_PAGE_HPTE_SUB@h
  498. ori r0,r0,_PAGE_HPTE_SUB@l
  499. andc r30,r30,r0
  500. b htab_insert_pte
  501. htab_bail_ok:
  502. li r3,0
  503. b htab_bail
  504. htab_pte_insert_ok:
  505. /* Insert slot number & secondary bit in PTE second half,
  506. * clear _PAGE_BUSY and set approriate HPTE slot bit
  507. */
  508. ld r6,STK_PARM(r6)(r1)
  509. li r0,_PAGE_BUSY
  510. andc r30,r30,r0
  511. /* HPTE SUB bit */
  512. li r0,1
  513. subfic r5,r25,27 /* Must match bit position in */
  514. sld r0,r0,r5 /* pgtable.h */
  515. or r30,r30,r0
  516. /* hindx */
  517. sldi r5,r25,2
  518. sld r3,r3,r5
  519. li r4,0xf
  520. sld r4,r4,r5
  521. andc r26,r26,r4
  522. or r26,r26,r3
  523. ori r5,r6,0x8000
  524. std r26,0(r5)
  525. lwsync
  526. std r30,0(r6)
  527. li r3, 0
  528. htab_bail:
  529. ld r25,STK_REG(r25)(r1)
  530. ld r26,STK_REG(r26)(r1)
  531. ld r27,STK_REG(r27)(r1)
  532. ld r28,STK_REG(r28)(r1)
  533. ld r29,STK_REG(r29)(r1)
  534. ld r30,STK_REG(r30)(r1)
  535. ld r31,STK_REG(r31)(r1)
  536. addi r1,r1,STACKFRAMESIZE
  537. ld r0,16(r1)
  538. mtlr r0
  539. blr
  540. htab_modify_pte:
  541. /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
  542. mr r4,r3
  543. sldi r5,r25,2
  544. srd r3,r26,r5
  545. /* Secondary group ? if yes, get a inverted hash value */
  546. mr r5,r28
  547. andi. r0,r3,0x8 /* page secondary ? */
  548. beq 1f
  549. not r5,r5
  550. 1: andi. r3,r3,0x7 /* extract idx alone */
  551. /* Calculate proper slot value for ppc_md.hpte_updatepp */
  552. and r0,r5,r27
  553. rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  554. add r3,r0,r3 /* add slot idx */
  555. /* Call ppc_md.hpte_updatepp */
  556. mr r5,r29 /* va */
  557. li r6,MMU_PAGE_4K /* page size */
  558. ld r7,STK_PARM(r9)(r1) /* segment size */
  559. ld r8,STK_PARM(r8)(r1) /* get "local" param */
  560. _GLOBAL(htab_call_hpte_updatepp)
  561. bl . /* patched by htab_finish_init() */
  562. /* if we failed because typically the HPTE wasn't really here
  563. * we try an insertion.
  564. */
  565. cmpdi 0,r3,-1
  566. beq- htab_insert_pte
  567. /* Clear the BUSY bit and Write out the PTE */
  568. li r0,_PAGE_BUSY
  569. andc r30,r30,r0
  570. ld r6,STK_PARM(r6)(r1)
  571. std r30,0(r6)
  572. li r3,0
  573. b htab_bail
  574. htab_wrong_access:
  575. /* Bail out clearing reservation */
  576. stdcx. r31,0,r6
  577. li r3,1
  578. b htab_bail
  579. htab_pte_insert_failure:
  580. /* Bail out restoring old PTE */
  581. ld r6,STK_PARM(r6)(r1)
  582. std r31,0(r6)
  583. li r3,-1
  584. b htab_bail
  585. #endif /* CONFIG_PPC_64K_PAGES */
  586. #ifdef CONFIG_PPC_HAS_HASH_64K
  587. /*****************************************************************************
  588. * *
  589. * 64K SW & 64K HW in a 64K segment pages implementation *
  590. * *
  591. *****************************************************************************/
  592. _GLOBAL(__hash_page_64K)
  593. mflr r0
  594. std r0,16(r1)
  595. stdu r1,-STACKFRAMESIZE(r1)
  596. /* Save all params that we need after a function call */
  597. std r6,STK_PARM(r6)(r1)
  598. std r8,STK_PARM(r8)(r1)
  599. std r9,STK_PARM(r9)(r1)
  600. /* Add _PAGE_PRESENT to access */
  601. ori r4,r4,_PAGE_PRESENT
  602. /* Save non-volatile registers.
  603. * r31 will hold "old PTE"
  604. * r30 is "new PTE"
  605. * r29 is "va"
  606. * r28 is a hash value
  607. * r27 is hashtab mask (maybe dynamic patched instead ?)
  608. */
  609. std r27,STK_REG(r27)(r1)
  610. std r28,STK_REG(r28)(r1)
  611. std r29,STK_REG(r29)(r1)
  612. std r30,STK_REG(r30)(r1)
  613. std r31,STK_REG(r31)(r1)
  614. /* Step 1:
  615. *
  616. * Check permissions, atomically mark the linux PTE busy
  617. * and hashed.
  618. */
  619. 1:
  620. ldarx r31,0,r6
  621. /* Check access rights (access & ~(pte_val(*ptep))) */
  622. andc. r0,r4,r31
  623. bne- ht64_wrong_access
  624. /* Check if PTE is busy */
  625. andi. r0,r31,_PAGE_BUSY
  626. /* If so, just bail out and refault if needed. Someone else
  627. * is changing this PTE anyway and might hash it.
  628. */
  629. bne- ht64_bail_ok
  630. BEGIN_FTR_SECTION
  631. /* Check if PTE has the cache-inhibit bit set */
  632. andi. r0,r31,_PAGE_NO_CACHE
  633. /* If so, bail out and refault as a 4k page */
  634. bne- ht64_bail_ok
  635. END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
  636. /* Prepare new PTE value (turn access RW into DIRTY, then
  637. * add BUSY and ACCESSED)
  638. */
  639. rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  640. or r30,r30,r31
  641. ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
  642. /* Write the linux PTE atomically (setting busy) */
  643. stdcx. r30,0,r6
  644. bne- 1b
  645. isync
  646. /* Step 2:
  647. *
  648. * Insert/Update the HPTE in the hash table. At this point,
  649. * r4 (access) is re-useable, we use it for the new HPTE flags
  650. */
  651. BEGIN_FTR_SECTION
  652. cmpdi r9,0 /* check segment size */
  653. bne 3f
  654. END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
  655. /* Calc va and put it in r29 */
  656. rldicr r29,r5,28,63-28
  657. rldicl r3,r3,0,36
  658. or r29,r3,r29
  659. /* Calculate hash value for primary slot and store it in r28 */
  660. rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
  661. rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
  662. xor r28,r5,r0
  663. b 4f
  664. 3: /* Calc VA and hash in r29 and r28 for 1T segment */
  665. sldi r29,r5,40 /* vsid << 40 */
  666. clrldi r3,r3,24 /* ea & 0xffffffffff */
  667. rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
  668. clrldi r5,r5,40 /* vsid & 0xffffff */
  669. rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
  670. xor r28,r28,r5
  671. or r29,r3,r29 /* VA */
  672. xor r28,r28,r0 /* hash */
  673. /* Convert linux PTE bits into HW equivalents */
  674. 4: andi. r3,r30,0x1fe /* Get basic set of flags */
  675. xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
  676. rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
  677. rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
  678. and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
  679. andc r0,r30,r0 /* r0 = pte & ~r0 */
  680. rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
  681. ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
  682. /* We eventually do the icache sync here (maybe inline that
  683. * code rather than call a C function...)
  684. */
  685. BEGIN_FTR_SECTION
  686. mr r4,r30
  687. mr r5,r7
  688. bl .hash_page_do_lazy_icache
  689. END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
  690. /* At this point, r3 contains new PP bits, save them in
  691. * place of "access" in the param area (sic)
  692. */
  693. std r3,STK_PARM(r4)(r1)
  694. /* Get htab_hash_mask */
  695. ld r4,htab_hash_mask@got(2)
  696. ld r27,0(r4) /* htab_hash_mask -> r27 */
  697. /* Check if we may already be in the hashtable, in this case, we
  698. * go to out-of-line code to try to modify the HPTE
  699. */
  700. rldicl. r0,r31,64-12,48
  701. bne ht64_modify_pte
  702. ht64_insert_pte:
  703. /* Clear hpte bits in new pte (we also clear BUSY btw) and
  704. * add _PAGE_HPTE_SUB0
  705. */
  706. lis r0,_PAGE_HPTEFLAGS@h
  707. ori r0,r0,_PAGE_HPTEFLAGS@l
  708. andc r30,r30,r0
  709. #ifdef CONFIG_PPC_64K_PAGES
  710. oris r30,r30,_PAGE_HPTE_SUB0@h
  711. #else
  712. ori r30,r30,_PAGE_HASHPTE
  713. #endif
  714. /* Phyical address in r5 */
  715. rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
  716. sldi r5,r5,PAGE_SHIFT
  717. /* Calculate primary group hash */
  718. and r0,r28,r27
  719. rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  720. /* Call ppc_md.hpte_insert */
  721. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  722. mr r4,r29 /* Retreive va */
  723. li r7,0 /* !bolted, !secondary */
  724. li r8,MMU_PAGE_64K
  725. ld r9,STK_PARM(r9)(r1) /* segment size */
  726. _GLOBAL(ht64_call_hpte_insert1)
  727. bl . /* patched by htab_finish_init() */
  728. cmpdi 0,r3,0
  729. bge ht64_pte_insert_ok /* Insertion successful */
  730. cmpdi 0,r3,-2 /* Critical failure */
  731. beq- ht64_pte_insert_failure
  732. /* Now try secondary slot */
  733. /* Phyical address in r5 */
  734. rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
  735. sldi r5,r5,PAGE_SHIFT
  736. /* Calculate secondary group hash */
  737. andc r0,r27,r28
  738. rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
  739. /* Call ppc_md.hpte_insert */
  740. ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
  741. mr r4,r29 /* Retreive va */
  742. li r7,HPTE_V_SECONDARY /* !bolted, secondary */
  743. li r8,MMU_PAGE_64K
  744. ld r9,STK_PARM(r9)(r1) /* segment size */
  745. _GLOBAL(ht64_call_hpte_insert2)
  746. bl . /* patched by htab_finish_init() */
  747. cmpdi 0,r3,0
  748. bge+ ht64_pte_insert_ok /* Insertion successful */
  749. cmpdi 0,r3,-2 /* Critical failure */
  750. beq- ht64_pte_insert_failure
  751. /* Both are full, we need to evict something */
  752. mftb r0
  753. /* Pick a random group based on TB */
  754. andi. r0,r0,1
  755. mr r5,r28
  756. bne 2f
  757. not r5,r5
  758. 2: and r0,r5,r27
  759. rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  760. /* Call ppc_md.hpte_remove */
  761. _GLOBAL(ht64_call_hpte_remove)
  762. bl . /* patched by htab_finish_init() */
  763. /* Try all again */
  764. b ht64_insert_pte
  765. ht64_bail_ok:
  766. li r3,0
  767. b ht64_bail
  768. ht64_pte_insert_ok:
  769. /* Insert slot number & secondary bit in PTE */
  770. rldimi r30,r3,12,63-15
  771. /* Write out the PTE with a normal write
  772. * (maybe add eieio may be good still ?)
  773. */
  774. ht64_write_out_pte:
  775. ld r6,STK_PARM(r6)(r1)
  776. std r30,0(r6)
  777. li r3, 0
  778. ht64_bail:
  779. ld r27,STK_REG(r27)(r1)
  780. ld r28,STK_REG(r28)(r1)
  781. ld r29,STK_REG(r29)(r1)
  782. ld r30,STK_REG(r30)(r1)
  783. ld r31,STK_REG(r31)(r1)
  784. addi r1,r1,STACKFRAMESIZE
  785. ld r0,16(r1)
  786. mtlr r0
  787. blr
  788. ht64_modify_pte:
  789. /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
  790. mr r4,r3
  791. rlwinm r3,r31,32-12,29,31
  792. /* Secondary group ? if yes, get a inverted hash value */
  793. mr r5,r28
  794. andi. r0,r31,_PAGE_F_SECOND
  795. beq 1f
  796. not r5,r5
  797. 1:
  798. /* Calculate proper slot value for ppc_md.hpte_updatepp */
  799. and r0,r5,r27
  800. rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
  801. add r3,r0,r3 /* add slot idx */
  802. /* Call ppc_md.hpte_updatepp */
  803. mr r5,r29 /* va */
  804. li r6,MMU_PAGE_64K
  805. ld r7,STK_PARM(r9)(r1) /* segment size */
  806. ld r8,STK_PARM(r8)(r1) /* get "local" param */
  807. _GLOBAL(ht64_call_hpte_updatepp)
  808. bl . /* patched by htab_finish_init() */
  809. /* if we failed because typically the HPTE wasn't really here
  810. * we try an insertion.
  811. */
  812. cmpdi 0,r3,-1
  813. beq- ht64_insert_pte
  814. /* Clear the BUSY bit and Write out the PTE */
  815. li r0,_PAGE_BUSY
  816. andc r30,r30,r0
  817. b ht64_write_out_pte
  818. ht64_wrong_access:
  819. /* Bail out clearing reservation */
  820. stdcx. r31,0,r6
  821. li r3,1
  822. b ht64_bail
  823. ht64_pte_insert_failure:
  824. /* Bail out restoring old PTE */
  825. ld r6,STK_PARM(r6)(r1)
  826. std r31,0(r6)
  827. li r3,-1
  828. b ht64_bail
  829. #endif /* CONFIG_PPC_HAS_HASH_64K */
  830. /*****************************************************************************
  831. * *
  832. * Huge pages implementation is in hugetlbpage.c *
  833. * *
  834. *****************************************************************************/