percpu.h 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. #ifndef __LINUX_PERCPU_H
  2. #define __LINUX_PERCPU_H
  3. #include <linux/preempt.h>
  4. #include <linux/slab.h> /* For kmalloc() */
  5. #include <linux/smp.h>
  6. #include <linux/cpumask.h>
  7. #include <linux/pfn.h>
  8. #include <asm/percpu.h>
  9. /* enough to cover all DEFINE_PER_CPUs in modules */
  10. #ifdef CONFIG_MODULES
  11. #define PERCPU_MODULE_RESERVE (8 << 10)
  12. #else
  13. #define PERCPU_MODULE_RESERVE 0
  14. #endif
  15. #ifndef PERCPU_ENOUGH_ROOM
  16. #define PERCPU_ENOUGH_ROOM \
  17. (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
  18. PERCPU_MODULE_RESERVE)
  19. #endif
  20. /*
  21. * Must be an lvalue. Since @var must be a simple identifier,
  22. * we force a syntax error here if it isn't.
  23. */
  24. #define get_cpu_var(var) (*({ \
  25. extern int simple_identifier_##var(void); \
  26. preempt_disable(); \
  27. &__get_cpu_var(var); }))
  28. #define put_cpu_var(var) preempt_enable()
  29. #ifdef CONFIG_SMP
  30. /* minimum unit size, also is the maximum supported allocation size */
  31. #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
  32. /*
  33. * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
  34. * back on the first chunk for dynamic percpu allocation if arch is
  35. * manually allocating and mapping it for faster access (as a part of
  36. * large page mapping for example).
  37. *
  38. * The following values give between one and two pages of free space
  39. * after typical minimal boot (2-way SMP, single disk and NIC) with
  40. * both defconfig and a distro config on x86_64 and 32. More
  41. * intelligent way to determine this would be nice.
  42. */
  43. #if BITS_PER_LONG > 32
  44. #define PERCPU_DYNAMIC_RESERVE (20 << 10)
  45. #else
  46. #define PERCPU_DYNAMIC_RESERVE (12 << 10)
  47. #endif
  48. extern void *pcpu_base_addr;
  49. extern const unsigned long *pcpu_unit_offsets;
  50. struct pcpu_group_info {
  51. int nr_units; /* aligned # of units */
  52. unsigned long base_offset; /* base address offset */
  53. unsigned int *cpu_map; /* unit->cpu map, empty
  54. * entries contain NR_CPUS */
  55. };
  56. struct pcpu_alloc_info {
  57. size_t static_size;
  58. size_t reserved_size;
  59. size_t dyn_size;
  60. size_t unit_size;
  61. size_t atom_size;
  62. size_t alloc_size;
  63. size_t __ai_size; /* internal, don't use */
  64. int nr_groups; /* 0 if grouping unnecessary */
  65. struct pcpu_group_info groups[];
  66. };
  67. enum pcpu_fc {
  68. PCPU_FC_AUTO,
  69. PCPU_FC_EMBED,
  70. PCPU_FC_PAGE,
  71. PCPU_FC_NR,
  72. };
  73. extern const char *pcpu_fc_names[PCPU_FC_NR];
  74. extern enum pcpu_fc pcpu_chosen_fc;
  75. typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
  76. size_t align);
  77. typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
  78. typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
  79. typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
  80. extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
  81. int nr_units);
  82. extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
  83. extern struct pcpu_alloc_info * __init pcpu_build_alloc_info(
  84. size_t reserved_size, ssize_t dyn_size,
  85. size_t atom_size,
  86. pcpu_fc_cpu_distance_fn_t cpu_distance_fn);
  87. extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
  88. void *base_addr);
  89. #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
  90. extern int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size,
  91. size_t atom_size,
  92. pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
  93. pcpu_fc_alloc_fn_t alloc_fn,
  94. pcpu_fc_free_fn_t free_fn);
  95. #endif
  96. #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
  97. extern int __init pcpu_page_first_chunk(size_t reserved_size,
  98. pcpu_fc_alloc_fn_t alloc_fn,
  99. pcpu_fc_free_fn_t free_fn,
  100. pcpu_fc_populate_pte_fn_t populate_pte_fn);
  101. #endif
  102. /*
  103. * Use this to get to a cpu's version of the per-cpu object
  104. * dynamically allocated. Non-atomic access to the current CPU's
  105. * version should probably be combined with get_cpu()/put_cpu().
  106. */
  107. #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
  108. extern void *__alloc_reserved_percpu(size_t size, size_t align);
  109. extern void *__alloc_percpu(size_t size, size_t align);
  110. extern void free_percpu(void *__pdata);
  111. #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
  112. extern void __init setup_per_cpu_areas(void);
  113. #endif
  114. #else /* CONFIG_SMP */
  115. #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  116. static inline void *__alloc_percpu(size_t size, size_t align)
  117. {
  118. /*
  119. * Can't easily make larger alignment work with kmalloc. WARN
  120. * on it. Larger alignment should only be used for module
  121. * percpu sections on SMP for which this path isn't used.
  122. */
  123. WARN_ON_ONCE(align > SMP_CACHE_BYTES);
  124. return kzalloc(size, GFP_KERNEL);
  125. }
  126. static inline void free_percpu(void *p)
  127. {
  128. kfree(p);
  129. }
  130. static inline void __init setup_per_cpu_areas(void) { }
  131. static inline void *pcpu_lpage_remapped(void *kaddr)
  132. {
  133. return NULL;
  134. }
  135. #endif /* CONFIG_SMP */
  136. #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type), \
  137. __alignof__(type))
  138. /*
  139. * Optional methods for optimized non-lvalue per-cpu variable access.
  140. *
  141. * @var can be a percpu variable or a field of it and its size should
  142. * equal char, int or long. percpu_read() evaluates to a lvalue and
  143. * all others to void.
  144. *
  145. * These operations are guaranteed to be atomic w.r.t. preemption.
  146. * The generic versions use plain get/put_cpu_var(). Archs are
  147. * encouraged to implement single-instruction alternatives which don't
  148. * require preemption protection.
  149. */
  150. #ifndef percpu_read
  151. # define percpu_read(var) \
  152. ({ \
  153. typeof(per_cpu_var(var)) __tmp_var__; \
  154. __tmp_var__ = get_cpu_var(var); \
  155. put_cpu_var(var); \
  156. __tmp_var__; \
  157. })
  158. #endif
  159. #define __percpu_generic_to_op(var, val, op) \
  160. do { \
  161. get_cpu_var(var) op val; \
  162. put_cpu_var(var); \
  163. } while (0)
  164. #ifndef percpu_write
  165. # define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
  166. #endif
  167. #ifndef percpu_add
  168. # define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
  169. #endif
  170. #ifndef percpu_sub
  171. # define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
  172. #endif
  173. #ifndef percpu_and
  174. # define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
  175. #endif
  176. #ifndef percpu_or
  177. # define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
  178. #endif
  179. #ifndef percpu_xor
  180. # define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
  181. #endif
  182. /*
  183. * Branching function to split up a function into a set of functions that
  184. * are called for different scalar sizes of the objects handled.
  185. */
  186. extern void __bad_size_call_parameter(void);
  187. #define __size_call_return(stem, variable) \
  188. ({ typeof(variable) ret__; \
  189. switch(sizeof(variable)) { \
  190. case 1: ret__ = stem##1(variable);break; \
  191. case 2: ret__ = stem##2(variable);break; \
  192. case 4: ret__ = stem##4(variable);break; \
  193. case 8: ret__ = stem##8(variable);break; \
  194. default: \
  195. __bad_size_call_parameter();break; \
  196. } \
  197. ret__; \
  198. })
  199. #define __size_call(stem, variable, ...) \
  200. do { \
  201. switch(sizeof(variable)) { \
  202. case 1: stem##1(variable, __VA_ARGS__);break; \
  203. case 2: stem##2(variable, __VA_ARGS__);break; \
  204. case 4: stem##4(variable, __VA_ARGS__);break; \
  205. case 8: stem##8(variable, __VA_ARGS__);break; \
  206. default: \
  207. __bad_size_call_parameter();break; \
  208. } \
  209. } while (0)
  210. /*
  211. * Optimized manipulation for memory allocated through the per cpu
  212. * allocator or for addresses of per cpu variables (can be determined
  213. * using per_cpu_var(xx).
  214. *
  215. * These operation guarantee exclusivity of access for other operations
  216. * on the *same* processor. The assumption is that per cpu data is only
  217. * accessed by a single processor instance (the current one).
  218. *
  219. * The first group is used for accesses that must be done in a
  220. * preemption safe way since we know that the context is not preempt
  221. * safe. Interrupts may occur. If the interrupt modifies the variable
  222. * too then RMW actions will not be reliable.
  223. *
  224. * The arch code can provide optimized functions in two ways:
  225. *
  226. * 1. Override the function completely. F.e. define this_cpu_add().
  227. * The arch must then ensure that the various scalar format passed
  228. * are handled correctly.
  229. *
  230. * 2. Provide functions for certain scalar sizes. F.e. provide
  231. * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
  232. * sized RMW actions. If arch code does not provide operations for
  233. * a scalar size then the fallback in the generic code will be
  234. * used.
  235. */
  236. #define _this_cpu_generic_read(pcp) \
  237. ({ typeof(pcp) ret__; \
  238. preempt_disable(); \
  239. ret__ = *this_cpu_ptr(&(pcp)); \
  240. preempt_enable(); \
  241. ret__; \
  242. })
  243. #ifndef this_cpu_read
  244. # ifndef this_cpu_read_1
  245. # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
  246. # endif
  247. # ifndef this_cpu_read_2
  248. # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
  249. # endif
  250. # ifndef this_cpu_read_4
  251. # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
  252. # endif
  253. # ifndef this_cpu_read_8
  254. # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
  255. # endif
  256. # define this_cpu_read(pcp) __size_call_return(this_cpu_read_, (pcp))
  257. #endif
  258. #define _this_cpu_generic_to_op(pcp, val, op) \
  259. do { \
  260. preempt_disable(); \
  261. *__this_cpu_ptr(&pcp) op val; \
  262. preempt_enable(); \
  263. } while (0)
  264. #ifndef this_cpu_write
  265. # ifndef this_cpu_write_1
  266. # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
  267. # endif
  268. # ifndef this_cpu_write_2
  269. # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
  270. # endif
  271. # ifndef this_cpu_write_4
  272. # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
  273. # endif
  274. # ifndef this_cpu_write_8
  275. # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
  276. # endif
  277. # define this_cpu_write(pcp, val) __size_call(this_cpu_write_, (pcp), (val))
  278. #endif
  279. #ifndef this_cpu_add
  280. # ifndef this_cpu_add_1
  281. # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
  282. # endif
  283. # ifndef this_cpu_add_2
  284. # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
  285. # endif
  286. # ifndef this_cpu_add_4
  287. # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
  288. # endif
  289. # ifndef this_cpu_add_8
  290. # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
  291. # endif
  292. # define this_cpu_add(pcp, val) __size_call(this_cpu_add_, (pcp), (val))
  293. #endif
  294. #ifndef this_cpu_sub
  295. # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val))
  296. #endif
  297. #ifndef this_cpu_inc
  298. # define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
  299. #endif
  300. #ifndef this_cpu_dec
  301. # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
  302. #endif
  303. #ifndef this_cpu_and
  304. # ifndef this_cpu_and_1
  305. # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
  306. # endif
  307. # ifndef this_cpu_and_2
  308. # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
  309. # endif
  310. # ifndef this_cpu_and_4
  311. # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
  312. # endif
  313. # ifndef this_cpu_and_8
  314. # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
  315. # endif
  316. # define this_cpu_and(pcp, val) __size_call(this_cpu_and_, (pcp), (val))
  317. #endif
  318. #ifndef this_cpu_or
  319. # ifndef this_cpu_or_1
  320. # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
  321. # endif
  322. # ifndef this_cpu_or_2
  323. # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
  324. # endif
  325. # ifndef this_cpu_or_4
  326. # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
  327. # endif
  328. # ifndef this_cpu_or_8
  329. # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
  330. # endif
  331. # define this_cpu_or(pcp, val) __size_call(this_cpu_or_, (pcp), (val))
  332. #endif
  333. #ifndef this_cpu_xor
  334. # ifndef this_cpu_xor_1
  335. # define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
  336. # endif
  337. # ifndef this_cpu_xor_2
  338. # define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
  339. # endif
  340. # ifndef this_cpu_xor_4
  341. # define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
  342. # endif
  343. # ifndef this_cpu_xor_8
  344. # define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
  345. # endif
  346. # define this_cpu_xor(pcp, val) __size_call(this_cpu_or_, (pcp), (val))
  347. #endif
  348. /*
  349. * Generic percpu operations that do not require preemption handling.
  350. * Either we do not care about races or the caller has the
  351. * responsibility of handling preemptions issues. Arch code can still
  352. * override these instructions since the arch per cpu code may be more
  353. * efficient and may actually get race freeness for free (that is the
  354. * case for x86 for example).
  355. *
  356. * If there is no other protection through preempt disable and/or
  357. * disabling interupts then one of these RMW operations can show unexpected
  358. * behavior because the execution thread was rescheduled on another processor
  359. * or an interrupt occurred and the same percpu variable was modified from
  360. * the interrupt context.
  361. */
  362. #ifndef __this_cpu_read
  363. # ifndef __this_cpu_read_1
  364. # define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
  365. # endif
  366. # ifndef __this_cpu_read_2
  367. # define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
  368. # endif
  369. # ifndef __this_cpu_read_4
  370. # define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
  371. # endif
  372. # ifndef __this_cpu_read_8
  373. # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
  374. # endif
  375. # define __this_cpu_read(pcp) __size_call_return(__this_cpu_read_, (pcp))
  376. #endif
  377. #define __this_cpu_generic_to_op(pcp, val, op) \
  378. do { \
  379. *__this_cpu_ptr(&(pcp)) op val; \
  380. } while (0)
  381. #ifndef __this_cpu_write
  382. # ifndef __this_cpu_write_1
  383. # define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
  384. # endif
  385. # ifndef __this_cpu_write_2
  386. # define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
  387. # endif
  388. # ifndef __this_cpu_write_4
  389. # define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
  390. # endif
  391. # ifndef __this_cpu_write_8
  392. # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
  393. # endif
  394. # define __this_cpu_write(pcp, val) __size_call(__this_cpu_write_, (pcp), (val))
  395. #endif
  396. #ifndef __this_cpu_add
  397. # ifndef __this_cpu_add_1
  398. # define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
  399. # endif
  400. # ifndef __this_cpu_add_2
  401. # define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
  402. # endif
  403. # ifndef __this_cpu_add_4
  404. # define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
  405. # endif
  406. # ifndef __this_cpu_add_8
  407. # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
  408. # endif
  409. # define __this_cpu_add(pcp, val) __size_call(__this_cpu_add_, (pcp), (val))
  410. #endif
  411. #ifndef __this_cpu_sub
  412. # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val))
  413. #endif
  414. #ifndef __this_cpu_inc
  415. # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
  416. #endif
  417. #ifndef __this_cpu_dec
  418. # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
  419. #endif
  420. #ifndef __this_cpu_and
  421. # ifndef __this_cpu_and_1
  422. # define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
  423. # endif
  424. # ifndef __this_cpu_and_2
  425. # define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
  426. # endif
  427. # ifndef __this_cpu_and_4
  428. # define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
  429. # endif
  430. # ifndef __this_cpu_and_8
  431. # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
  432. # endif
  433. # define __this_cpu_and(pcp, val) __size_call(__this_cpu_and_, (pcp), (val))
  434. #endif
  435. #ifndef __this_cpu_or
  436. # ifndef __this_cpu_or_1
  437. # define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
  438. # endif
  439. # ifndef __this_cpu_or_2
  440. # define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
  441. # endif
  442. # ifndef __this_cpu_or_4
  443. # define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
  444. # endif
  445. # ifndef __this_cpu_or_8
  446. # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
  447. # endif
  448. # define __this_cpu_or(pcp, val) __size_call(__this_cpu_or_, (pcp), (val))
  449. #endif
  450. #ifndef __this_cpu_xor
  451. # ifndef __this_cpu_xor_1
  452. # define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
  453. # endif
  454. # ifndef __this_cpu_xor_2
  455. # define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
  456. # endif
  457. # ifndef __this_cpu_xor_4
  458. # define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
  459. # endif
  460. # ifndef __this_cpu_xor_8
  461. # define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
  462. # endif
  463. # define __this_cpu_xor(pcp, val) __size_call(__this_cpu_xor_, (pcp), (val))
  464. #endif
  465. /*
  466. * IRQ safe versions of the per cpu RMW operations. Note that these operations
  467. * are *not* safe against modification of the same variable from another
  468. * processors (which one gets when using regular atomic operations)
  469. . They are guaranteed to be atomic vs. local interrupts and
  470. * preemption only.
  471. */
  472. #define irqsafe_cpu_generic_to_op(pcp, val, op) \
  473. do { \
  474. unsigned long flags; \
  475. local_irq_save(flags); \
  476. *__this_cpu_ptr(&(pcp)) op val; \
  477. local_irq_restore(flags); \
  478. } while (0)
  479. #ifndef irqsafe_cpu_add
  480. # ifndef irqsafe_cpu_add_1
  481. # define irqsafe_cpu_add_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
  482. # endif
  483. # ifndef irqsafe_cpu_add_2
  484. # define irqsafe_cpu_add_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
  485. # endif
  486. # ifndef irqsafe_cpu_add_4
  487. # define irqsafe_cpu_add_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
  488. # endif
  489. # ifndef irqsafe_cpu_add_8
  490. # define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
  491. # endif
  492. # define irqsafe_cpu_add(pcp, val) __size_call(irqsafe_cpu_add_, (pcp), (val))
  493. #endif
  494. #ifndef irqsafe_cpu_sub
  495. # define irqsafe_cpu_sub(pcp, val) irqsafe_cpu_add((pcp), -(val))
  496. #endif
  497. #ifndef irqsafe_cpu_inc
  498. # define irqsafe_cpu_inc(pcp) irqsafe_cpu_add((pcp), 1)
  499. #endif
  500. #ifndef irqsafe_cpu_dec
  501. # define irqsafe_cpu_dec(pcp) irqsafe_cpu_sub((pcp), 1)
  502. #endif
  503. #ifndef irqsafe_cpu_and
  504. # ifndef irqsafe_cpu_and_1
  505. # define irqsafe_cpu_and_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
  506. # endif
  507. # ifndef irqsafe_cpu_and_2
  508. # define irqsafe_cpu_and_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
  509. # endif
  510. # ifndef irqsafe_cpu_and_4
  511. # define irqsafe_cpu_and_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
  512. # endif
  513. # ifndef irqsafe_cpu_and_8
  514. # define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
  515. # endif
  516. # define irqsafe_cpu_and(pcp, val) __size_call(irqsafe_cpu_and_, (val))
  517. #endif
  518. #ifndef irqsafe_cpu_or
  519. # ifndef irqsafe_cpu_or_1
  520. # define irqsafe_cpu_or_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
  521. # endif
  522. # ifndef irqsafe_cpu_or_2
  523. # define irqsafe_cpu_or_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
  524. # endif
  525. # ifndef irqsafe_cpu_or_4
  526. # define irqsafe_cpu_or_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
  527. # endif
  528. # ifndef irqsafe_cpu_or_8
  529. # define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
  530. # endif
  531. # define irqsafe_cpu_or(pcp, val) __size_call(irqsafe_cpu_or_, (val))
  532. #endif
  533. #ifndef irqsafe_cpu_xor
  534. # ifndef irqsafe_cpu_xor_1
  535. # define irqsafe_cpu_xor_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
  536. # endif
  537. # ifndef irqsafe_cpu_xor_2
  538. # define irqsafe_cpu_xor_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
  539. # endif
  540. # ifndef irqsafe_cpu_xor_4
  541. # define irqsafe_cpu_xor_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
  542. # endif
  543. # ifndef irqsafe_cpu_xor_8
  544. # define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
  545. # endif
  546. # define irqsafe_cpu_xor(pcp, val) __size_call(irqsafe_cpu_xor_, (val))
  547. #endif
  548. #endif /* __LINUX_PERCPU_H */