|
@@ -33,60 +33,47 @@ extern void __add_wrong_size(void)
|
|
|
#define __X86_CASE_Q -1 /* sizeof will never return -1 */
|
|
|
#endif
|
|
|
|
|
|
+/*
|
|
|
+ * An exchange-type operation, which takes a value and a pointer, and
|
|
|
+ * returns a the old value.
|
|
|
+ */
|
|
|
+#define __xchg_op(ptr, arg, op, lock) \
|
|
|
+ ({ \
|
|
|
+ __typeof__ (*(ptr)) __ret = (arg); \
|
|
|
+ switch (sizeof(*(ptr))) { \
|
|
|
+ case __X86_CASE_B: \
|
|
|
+ asm volatile (lock #op "b %b0, %1\n" \
|
|
|
+ : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
+ : : "memory", "cc"); \
|
|
|
+ break; \
|
|
|
+ case __X86_CASE_W: \
|
|
|
+ asm volatile (lock #op "w %w0, %1\n" \
|
|
|
+ : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
+ : : "memory", "cc"); \
|
|
|
+ break; \
|
|
|
+ case __X86_CASE_L: \
|
|
|
+ asm volatile (lock #op "l %0, %1\n" \
|
|
|
+ : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
+ : : "memory", "cc"); \
|
|
|
+ break; \
|
|
|
+ case __X86_CASE_Q: \
|
|
|
+ asm volatile (lock #op "q %q0, %1\n" \
|
|
|
+ : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
+ : : "memory", "cc"); \
|
|
|
+ break; \
|
|
|
+ default: \
|
|
|
+ __ ## op ## _wrong_size(); \
|
|
|
+ } \
|
|
|
+ __ret; \
|
|
|
+ })
|
|
|
+
|
|
|
/*
|
|
|
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
|
|
|
* Since this is generally used to protect other memory information, we
|
|
|
* use "asm volatile" and "memory" clobbers to prevent gcc from moving
|
|
|
* information around.
|
|
|
*/
|
|
|
-#define __xchg(x, ptr, size) \
|
|
|
-({ \
|
|
|
- __typeof(*(ptr)) __x = (x); \
|
|
|
- switch (size) { \
|
|
|
- case __X86_CASE_B: \
|
|
|
- { \
|
|
|
- volatile u8 *__ptr = (volatile u8 *)(ptr); \
|
|
|
- asm volatile("xchgb %0,%1" \
|
|
|
- : "=q" (__x), "+m" (*__ptr) \
|
|
|
- : "0" (__x) \
|
|
|
- : "memory"); \
|
|
|
- break; \
|
|
|
- } \
|
|
|
- case __X86_CASE_W: \
|
|
|
- { \
|
|
|
- volatile u16 *__ptr = (volatile u16 *)(ptr); \
|
|
|
- asm volatile("xchgw %0,%1" \
|
|
|
- : "=r" (__x), "+m" (*__ptr) \
|
|
|
- : "0" (__x) \
|
|
|
- : "memory"); \
|
|
|
- break; \
|
|
|
- } \
|
|
|
- case __X86_CASE_L: \
|
|
|
- { \
|
|
|
- volatile u32 *__ptr = (volatile u32 *)(ptr); \
|
|
|
- asm volatile("xchgl %0,%1" \
|
|
|
- : "=r" (__x), "+m" (*__ptr) \
|
|
|
- : "0" (__x) \
|
|
|
- : "memory"); \
|
|
|
- break; \
|
|
|
- } \
|
|
|
- case __X86_CASE_Q: \
|
|
|
- { \
|
|
|
- volatile u64 *__ptr = (volatile u64 *)(ptr); \
|
|
|
- asm volatile("xchgq %0,%1" \
|
|
|
- : "=r" (__x), "+m" (*__ptr) \
|
|
|
- : "0" (__x) \
|
|
|
- : "memory"); \
|
|
|
- break; \
|
|
|
- } \
|
|
|
- default: \
|
|
|
- __xchg_wrong_size(); \
|
|
|
- } \
|
|
|
- __x; \
|
|
|
-})
|
|
|
-
|
|
|
-#define xchg(ptr, v) \
|
|
|
- __xchg((v), (ptr), sizeof(*ptr))
|
|
|
+#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "")
|
|
|
|
|
|
/*
|
|
|
* Atomic compare and exchange. Compare OLD with MEM, if identical,
|
|
@@ -167,36 +154,6 @@ extern void __add_wrong_size(void)
|
|
|
__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
|
|
|
#endif
|
|
|
|
|
|
-#define __xadd(ptr, inc, lock) \
|
|
|
- ({ \
|
|
|
- __typeof__ (*(ptr)) __ret = (inc); \
|
|
|
- switch (sizeof(*(ptr))) { \
|
|
|
- case __X86_CASE_B: \
|
|
|
- asm volatile (lock "xaddb %b0, %1\n" \
|
|
|
- : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
- : : "memory", "cc"); \
|
|
|
- break; \
|
|
|
- case __X86_CASE_W: \
|
|
|
- asm volatile (lock "xaddw %w0, %1\n" \
|
|
|
- : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
- : : "memory", "cc"); \
|
|
|
- break; \
|
|
|
- case __X86_CASE_L: \
|
|
|
- asm volatile (lock "xaddl %0, %1\n" \
|
|
|
- : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
- : : "memory", "cc"); \
|
|
|
- break; \
|
|
|
- case __X86_CASE_Q: \
|
|
|
- asm volatile (lock "xaddq %q0, %1\n" \
|
|
|
- : "+r" (__ret), "+m" (*(ptr)) \
|
|
|
- : : "memory", "cc"); \
|
|
|
- break; \
|
|
|
- default: \
|
|
|
- __xadd_wrong_size(); \
|
|
|
- } \
|
|
|
- __ret; \
|
|
|
- })
|
|
|
-
|
|
|
/*
|
|
|
* xadd() adds "inc" to "*ptr" and atomically returns the previous
|
|
|
* value of "*ptr".
|
|
@@ -205,6 +162,7 @@ extern void __add_wrong_size(void)
|
|
|
* xadd_sync() is always locked
|
|
|
* xadd_local() is never locked
|
|
|
*/
|
|
|
+#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
|
|
|
#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
|
|
|
#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
|
|
|
#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
|