spinlock.h 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. #ifndef _ASM_X86_SPINLOCK_H
  2. #define _ASM_X86_SPINLOCK_H
  3. #include <linux/atomic.h>
  4. #include <asm/page.h>
  5. #include <asm/processor.h>
  6. #include <linux/compiler.h>
  7. #include <asm/paravirt.h>
  8. /*
  9. * Your basic SMP spinlocks, allowing only a single CPU anywhere
  10. *
  11. * Simple spin lock operations. There are two variants, one clears IRQ's
  12. * on the local processor, one does not.
  13. *
  14. * These are fair FIFO ticket locks, which are currently limited to 256
  15. * CPUs.
  16. *
  17. * (the type definitions are in asm/spinlock_types.h)
  18. */
  19. #ifdef CONFIG_X86_32
  20. # define LOCK_PTR_REG "a"
  21. # define REG_PTR_MODE "k"
  22. #else
  23. # define LOCK_PTR_REG "D"
  24. # define REG_PTR_MODE "q"
  25. #endif
  26. #if defined(CONFIG_X86_32) && \
  27. (defined(CONFIG_X86_OOSTORE) || defined(CONFIG_X86_PPRO_FENCE))
  28. /*
  29. * On PPro SMP or if we are using OOSTORE, we use a locked operation to unlock
  30. * (PPro errata 66, 92)
  31. */
  32. # define UNLOCK_LOCK_PREFIX LOCK_PREFIX
  33. #else
  34. # define UNLOCK_LOCK_PREFIX
  35. #endif
  36. /*
  37. * Ticket locks are conceptually two parts, one indicating the current head of
  38. * the queue, and the other indicating the current tail. The lock is acquired
  39. * by atomically noting the tail and incrementing it by one (thus adding
  40. * ourself to the queue and noting our position), then waiting until the head
  41. * becomes equal to the the initial value of the tail.
  42. *
  43. * We use an xadd covering *both* parts of the lock, to increment the tail and
  44. * also load the position of the head, which takes care of memory ordering
  45. * issues and should be optimal for the uncontended case. Note the tail must be
  46. * in the high part, because a wide xadd increment of the low part would carry
  47. * up and contaminate the high part.
  48. *
  49. * With fewer than 2^8 possible CPUs, we can use x86's partial registers to
  50. * save some instructions and make the code more elegant. There really isn't
  51. * much between them in performance though, especially as locks are out of line.
  52. */
  53. #if (NR_CPUS < 256)
  54. #define TICKET_SHIFT 8
  55. static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
  56. {
  57. short inc = 0x0100;
  58. asm volatile (
  59. LOCK_PREFIX "xaddw %w0, %1\n"
  60. "1:\t"
  61. "cmpb %h0, %b0\n\t"
  62. "je 2f\n\t"
  63. "rep ; nop\n\t"
  64. "movb %1, %b0\n\t"
  65. /* don't need lfence here, because loads are in-order */
  66. "jmp 1b\n"
  67. "2:"
  68. : "+Q" (inc), "+m" (lock->slock)
  69. :
  70. : "memory", "cc");
  71. }
  72. static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
  73. {
  74. int tmp, new;
  75. asm volatile("movzwl %2, %0\n\t"
  76. "cmpb %h0,%b0\n\t"
  77. "leal 0x100(%" REG_PTR_MODE "0), %1\n\t"
  78. "jne 1f\n\t"
  79. LOCK_PREFIX "cmpxchgw %w1,%2\n\t"
  80. "1:"
  81. "sete %b1\n\t"
  82. "movzbl %b1,%0\n\t"
  83. : "=&a" (tmp), "=&q" (new), "+m" (lock->slock)
  84. :
  85. : "memory", "cc");
  86. return tmp;
  87. }
  88. static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
  89. {
  90. asm volatile(UNLOCK_LOCK_PREFIX "incb %0"
  91. : "+m" (lock->slock)
  92. :
  93. : "memory", "cc");
  94. }
  95. #else
  96. #define TICKET_SHIFT 16
  97. static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
  98. {
  99. int inc = 0x00010000;
  100. int tmp;
  101. asm volatile(LOCK_PREFIX "xaddl %0, %1\n"
  102. "movzwl %w0, %2\n\t"
  103. "shrl $16, %0\n\t"
  104. "1:\t"
  105. "cmpl %0, %2\n\t"
  106. "je 2f\n\t"
  107. "rep ; nop\n\t"
  108. "movzwl %1, %2\n\t"
  109. /* don't need lfence here, because loads are in-order */
  110. "jmp 1b\n"
  111. "2:"
  112. : "+r" (inc), "+m" (lock->slock), "=&r" (tmp)
  113. :
  114. : "memory", "cc");
  115. }
  116. static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
  117. {
  118. int tmp;
  119. int new;
  120. asm volatile("movl %2,%0\n\t"
  121. "movl %0,%1\n\t"
  122. "roll $16, %0\n\t"
  123. "cmpl %0,%1\n\t"
  124. "leal 0x00010000(%" REG_PTR_MODE "0), %1\n\t"
  125. "jne 1f\n\t"
  126. LOCK_PREFIX "cmpxchgl %1,%2\n\t"
  127. "1:"
  128. "sete %b1\n\t"
  129. "movzbl %b1,%0\n\t"
  130. : "=&a" (tmp), "=&q" (new), "+m" (lock->slock)
  131. :
  132. : "memory", "cc");
  133. return tmp;
  134. }
  135. static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
  136. {
  137. asm volatile(UNLOCK_LOCK_PREFIX "incw %0"
  138. : "+m" (lock->slock)
  139. :
  140. : "memory", "cc");
  141. }
  142. #endif
  143. static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)
  144. {
  145. int tmp = ACCESS_ONCE(lock->slock);
  146. return !!(((tmp >> TICKET_SHIFT) ^ tmp) & ((1 << TICKET_SHIFT) - 1));
  147. }
  148. static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)
  149. {
  150. int tmp = ACCESS_ONCE(lock->slock);
  151. return (((tmp >> TICKET_SHIFT) - tmp) & ((1 << TICKET_SHIFT) - 1)) > 1;
  152. }
  153. #ifndef CONFIG_PARAVIRT_SPINLOCKS
  154. static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  155. {
  156. return __ticket_spin_is_locked(lock);
  157. }
  158. static inline int arch_spin_is_contended(arch_spinlock_t *lock)
  159. {
  160. return __ticket_spin_is_contended(lock);
  161. }
  162. #define arch_spin_is_contended arch_spin_is_contended
  163. static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
  164. {
  165. __ticket_spin_lock(lock);
  166. }
  167. static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
  168. {
  169. return __ticket_spin_trylock(lock);
  170. }
  171. static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
  172. {
  173. __ticket_spin_unlock(lock);
  174. }
  175. static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
  176. unsigned long flags)
  177. {
  178. arch_spin_lock(lock);
  179. }
  180. #endif /* CONFIG_PARAVIRT_SPINLOCKS */
  181. static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
  182. {
  183. while (arch_spin_is_locked(lock))
  184. cpu_relax();
  185. }
  186. /*
  187. * Read-write spinlocks, allowing multiple readers
  188. * but only one writer.
  189. *
  190. * NOTE! it is quite common to have readers in interrupts
  191. * but no interrupt writers. For those circumstances we
  192. * can "mix" irq-safe locks - any writer needs to get a
  193. * irq-safe write-lock, but readers can get non-irqsafe
  194. * read-locks.
  195. *
  196. * On x86, we implement read-write locks as a 32-bit counter
  197. * with the high bit (sign) being the "contended" bit.
  198. */
  199. /**
  200. * read_can_lock - would read_trylock() succeed?
  201. * @lock: the rwlock in question.
  202. */
  203. static inline int arch_read_can_lock(arch_rwlock_t *lock)
  204. {
  205. return lock->lock > 0;
  206. }
  207. /**
  208. * write_can_lock - would write_trylock() succeed?
  209. * @lock: the rwlock in question.
  210. */
  211. static inline int arch_write_can_lock(arch_rwlock_t *lock)
  212. {
  213. return lock->write == WRITE_LOCK_CMP;
  214. }
  215. static inline void arch_read_lock(arch_rwlock_t *rw)
  216. {
  217. asm volatile(LOCK_PREFIX READ_LOCK_SIZE(dec) " (%0)\n\t"
  218. "jns 1f\n"
  219. "call __read_lock_failed\n\t"
  220. "1:\n"
  221. ::LOCK_PTR_REG (rw) : "memory");
  222. }
  223. static inline void arch_write_lock(arch_rwlock_t *rw)
  224. {
  225. asm volatile(LOCK_PREFIX WRITE_LOCK_SUB(%1) "(%0)\n\t"
  226. "jz 1f\n"
  227. "call __write_lock_failed\n\t"
  228. "1:\n"
  229. ::LOCK_PTR_REG (&rw->write), "i" (RW_LOCK_BIAS)
  230. : "memory");
  231. }
  232. static inline int arch_read_trylock(arch_rwlock_t *lock)
  233. {
  234. READ_LOCK_ATOMIC(t) *count = (READ_LOCK_ATOMIC(t) *)lock;
  235. if (READ_LOCK_ATOMIC(dec_return)(count) >= 0)
  236. return 1;
  237. READ_LOCK_ATOMIC(inc)(count);
  238. return 0;
  239. }
  240. static inline int arch_write_trylock(arch_rwlock_t *lock)
  241. {
  242. atomic_t *count = (atomic_t *)&lock->write;
  243. if (atomic_sub_and_test(WRITE_LOCK_CMP, count))
  244. return 1;
  245. atomic_add(WRITE_LOCK_CMP, count);
  246. return 0;
  247. }
  248. static inline void arch_read_unlock(arch_rwlock_t *rw)
  249. {
  250. asm volatile(LOCK_PREFIX READ_LOCK_SIZE(inc) " %0"
  251. :"+m" (rw->lock) : : "memory");
  252. }
  253. static inline void arch_write_unlock(arch_rwlock_t *rw)
  254. {
  255. asm volatile(LOCK_PREFIX WRITE_LOCK_ADD(%1) "%0"
  256. : "+m" (rw->write) : "i" (RW_LOCK_BIAS) : "memory");
  257. }
  258. #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
  259. #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
  260. #undef READ_LOCK_SIZE
  261. #undef READ_LOCK_ATOMIC
  262. #undef WRITE_LOCK_ADD
  263. #undef WRITE_LOCK_SUB
  264. #undef WRITE_LOCK_CMP
  265. #define arch_spin_relax(lock) cpu_relax()
  266. #define arch_read_relax(lock) cpu_relax()
  267. #define arch_write_relax(lock) cpu_relax()
  268. /* The {read|write|spin}_lock() on x86 are full memory barriers. */
  269. static inline void smp_mb__after_lock(void) { }
  270. #define ARCH_HAS_SMP_MB_AFTER_LOCK
  271. #endif /* _ASM_X86_SPINLOCK_H */