mcpm_head.S 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. /*
  2. * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
  3. *
  4. * Created by: Nicolas Pitre, March 2012
  5. * Copyright: (C) 2012-2013 Linaro Limited
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as
  9. * published by the Free Software Foundation.
  10. *
  11. *
  12. * Refer to Documentation/arm/cluster-pm-race-avoidance.txt
  13. * for details of the synchronisation algorithms used here.
  14. */
  15. #include <linux/linkage.h>
  16. #include <asm/mcpm.h>
  17. #include "vlock.h"
  18. .if MCPM_SYNC_CLUSTER_CPUS
  19. .error "cpus must be the first member of struct mcpm_sync_struct"
  20. .endif
  21. .macro pr_dbg string
  22. #if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
  23. b 1901f
  24. 1902: .asciz "CPU"
  25. 1903: .asciz " cluster"
  26. 1904: .asciz ": \string"
  27. .align
  28. 1901: adr r0, 1902b
  29. bl printascii
  30. mov r0, r9
  31. bl printhex2
  32. adr r0, 1903b
  33. bl printascii
  34. mov r0, r10
  35. bl printhex2
  36. adr r0, 1904b
  37. bl printascii
  38. #endif
  39. .endm
  40. .arm
  41. .align
  42. ENTRY(mcpm_entry_point)
  43. THUMB( adr r12, BSYM(1f) )
  44. THUMB( bx r12 )
  45. THUMB( .thumb )
  46. 1:
  47. mrc p15, 0, r0, c0, c0, 5 @ MPIDR
  48. ubfx r9, r0, #0, #8 @ r9 = cpu
  49. ubfx r10, r0, #8, #8 @ r10 = cluster
  50. mov r3, #MAX_CPUS_PER_CLUSTER
  51. mla r4, r3, r10, r9 @ r4 = canonical CPU index
  52. cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
  53. blo 2f
  54. /* We didn't expect this CPU. Try to cheaply make it quiet. */
  55. 1: wfi
  56. wfe
  57. b 1b
  58. 2: pr_dbg "kernel mcpm_entry_point\n"
  59. /*
  60. * MMU is off so we need to get to various variables in a
  61. * position independent way.
  62. */
  63. adr r5, 3f
  64. ldmia r5, {r6, r7, r8, r11}
  65. add r6, r5, r6 @ r6 = mcpm_entry_vectors
  66. ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys
  67. add r8, r5, r8 @ r8 = mcpm_sync
  68. add r11, r5, r11 @ r11 = first_man_locks
  69. mov r0, #MCPM_SYNC_CLUSTER_SIZE
  70. mla r8, r0, r10, r8 @ r8 = sync cluster base
  71. @ Signal that this CPU is coming UP:
  72. mov r0, #CPU_COMING_UP
  73. mov r5, #MCPM_SYNC_CPU_SIZE
  74. mla r5, r9, r5, r8 @ r5 = sync cpu address
  75. strb r0, [r5]
  76. @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
  77. @ state, because there is at least one active CPU (this CPU).
  78. mov r0, #VLOCK_SIZE
  79. mla r11, r0, r10, r11 @ r11 = cluster first man lock
  80. mov r0, r11
  81. mov r1, r9 @ cpu
  82. bl vlock_trylock @ implies DMB
  83. cmp r0, #0 @ failed to get the lock?
  84. bne mcpm_setup_wait @ wait for cluster setup if so
  85. ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
  86. cmp r0, #CLUSTER_UP @ cluster already up?
  87. bne mcpm_setup @ if not, set up the cluster
  88. @ Otherwise, release the first man lock and skip setup:
  89. mov r0, r11
  90. bl vlock_unlock
  91. b mcpm_setup_complete
  92. mcpm_setup:
  93. @ Control dependency implies strb not observable before previous ldrb.
  94. @ Signal that the cluster is being brought up:
  95. mov r0, #INBOUND_COMING_UP
  96. strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
  97. dmb
  98. @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
  99. @ point onwards will observe INBOUND_COMING_UP and abort.
  100. @ Wait for any previously-pending cluster teardown operations to abort
  101. @ or complete:
  102. mcpm_teardown_wait:
  103. ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
  104. cmp r0, #CLUSTER_GOING_DOWN
  105. bne first_man_setup
  106. wfe
  107. b mcpm_teardown_wait
  108. first_man_setup:
  109. dmb
  110. @ If the outbound gave up before teardown started, skip cluster setup:
  111. cmp r0, #CLUSTER_UP
  112. beq mcpm_setup_leave
  113. @ power_up_setup is now responsible for setting up the cluster:
  114. cmp r7, #0
  115. mov r0, #1 @ second (cluster) affinity level
  116. blxne r7 @ Call power_up_setup if defined
  117. dmb
  118. mov r0, #CLUSTER_UP
  119. strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
  120. dmb
  121. mcpm_setup_leave:
  122. @ Leave the cluster setup critical section:
  123. mov r0, #INBOUND_NOT_COMING_UP
  124. strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
  125. dsb
  126. sev
  127. mov r0, r11
  128. bl vlock_unlock @ implies DMB
  129. b mcpm_setup_complete
  130. @ In the contended case, non-first men wait here for cluster setup
  131. @ to complete:
  132. mcpm_setup_wait:
  133. ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
  134. cmp r0, #CLUSTER_UP
  135. wfene
  136. bne mcpm_setup_wait
  137. dmb
  138. mcpm_setup_complete:
  139. @ If a platform-specific CPU setup hook is needed, it is
  140. @ called from here.
  141. cmp r7, #0
  142. mov r0, #0 @ first (CPU) affinity level
  143. blxne r7 @ Call power_up_setup if defined
  144. dmb
  145. @ Mark the CPU as up:
  146. mov r0, #CPU_UP
  147. strb r0, [r5]
  148. @ Observability order of CPU_UP and opening of the gate does not matter.
  149. mcpm_entry_gated:
  150. ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector
  151. cmp r5, #0
  152. wfeeq
  153. beq mcpm_entry_gated
  154. dmb
  155. pr_dbg "released\n"
  156. bx r5
  157. .align 2
  158. 3: .word mcpm_entry_vectors - .
  159. .word mcpm_power_up_setup_phys - 3b
  160. .word mcpm_sync - 3b
  161. .word first_man_locks - 3b
  162. ENDPROC(mcpm_entry_point)
  163. .bss
  164. .align CACHE_WRITEBACK_ORDER
  165. .type first_man_locks, #object
  166. first_man_locks:
  167. .space VLOCK_SIZE * MAX_NR_CLUSTERS
  168. .align CACHE_WRITEBACK_ORDER
  169. .type mcpm_entry_vectors, #object
  170. ENTRY(mcpm_entry_vectors)
  171. .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
  172. .type mcpm_power_up_setup_phys, #object
  173. ENTRY(mcpm_power_up_setup_phys)
  174. .space 4 @ set by mcpm_sync_init()