rcupdate.h 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003
  1. /*
  2. * Read-Copy Update mechanism for mutual exclusion
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation; either version 2 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * Copyright IBM Corporation, 2001
  19. *
  20. * Author: Dipankar Sarma <dipankar@in.ibm.com>
  21. *
  22. * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
  23. * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
  24. * Papers:
  25. * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
  26. * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
  27. *
  28. * For detailed explanation of Read-Copy Update mechanism see -
  29. * http://lse.sourceforge.net/locking/rcupdate.html
  30. *
  31. */
  32. #ifndef __LINUX_RCUPDATE_H
  33. #define __LINUX_RCUPDATE_H
  34. #include <linux/types.h>
  35. #include <linux/cache.h>
  36. #include <linux/spinlock.h>
  37. #include <linux/threads.h>
  38. #include <linux/cpumask.h>
  39. #include <linux/seqlock.h>
  40. #include <linux/lockdep.h>
  41. #include <linux/completion.h>
  42. #include <linux/debugobjects.h>
  43. #include <linux/bug.h>
  44. #include <linux/compiler.h>
  45. #ifdef CONFIG_RCU_TORTURE_TEST
  46. extern int rcutorture_runnable; /* for sysctl */
  47. #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
  48. #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
  49. extern void rcutorture_record_test_transition(void);
  50. extern void rcutorture_record_progress(unsigned long vernum);
  51. extern void do_trace_rcu_torture_read(char *rcutorturename,
  52. struct rcu_head *rhp,
  53. unsigned long secs,
  54. unsigned long c_old,
  55. unsigned long c);
  56. #else
  57. static inline void rcutorture_record_test_transition(void)
  58. {
  59. }
  60. static inline void rcutorture_record_progress(unsigned long vernum)
  61. {
  62. }
  63. #ifdef CONFIG_RCU_TRACE
  64. extern void do_trace_rcu_torture_read(char *rcutorturename,
  65. struct rcu_head *rhp,
  66. unsigned long secs,
  67. unsigned long c_old,
  68. unsigned long c);
  69. #else
  70. #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
  71. do { } while (0)
  72. #endif
  73. #endif
  74. #define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b))
  75. #define UINT_CMP_LT(a, b) (UINT_MAX / 2 < (a) - (b))
  76. #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
  77. #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
  78. #define ulong2long(a) (*(long *)(&(a)))
  79. /* Exported common interfaces */
  80. #ifdef CONFIG_PREEMPT_RCU
  81. /**
  82. * call_rcu() - Queue an RCU callback for invocation after a grace period.
  83. * @head: structure to be used for queueing the RCU updates.
  84. * @func: actual callback function to be invoked after the grace period
  85. *
  86. * The callback function will be invoked some time after a full grace
  87. * period elapses, in other words after all pre-existing RCU read-side
  88. * critical sections have completed. However, the callback function
  89. * might well execute concurrently with RCU read-side critical sections
  90. * that started after call_rcu() was invoked. RCU read-side critical
  91. * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
  92. * and may be nested.
  93. *
  94. * Note that all CPUs must agree that the grace period extended beyond
  95. * all pre-existing RCU read-side critical section. On systems with more
  96. * than one CPU, this means that when "func()" is invoked, each CPU is
  97. * guaranteed to have executed a full memory barrier since the end of its
  98. * last RCU read-side critical section whose beginning preceded the call
  99. * to call_rcu(). It also means that each CPU executing an RCU read-side
  100. * critical section that continues beyond the start of "func()" must have
  101. * executed a memory barrier after the call_rcu() but before the beginning
  102. * of that RCU read-side critical section. Note that these guarantees
  103. * include CPUs that are offline, idle, or executing in user mode, as
  104. * well as CPUs that are executing in the kernel.
  105. *
  106. * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
  107. * resulting RCU callback function "func()", then both CPU A and CPU B are
  108. * guaranteed to execute a full memory barrier during the time interval
  109. * between the call to call_rcu() and the invocation of "func()" -- even
  110. * if CPU A and CPU B are the same CPU (but again only if the system has
  111. * more than one CPU).
  112. */
  113. extern void call_rcu(struct rcu_head *head,
  114. void (*func)(struct rcu_head *head));
  115. #else /* #ifdef CONFIG_PREEMPT_RCU */
  116. /* In classic RCU, call_rcu() is just call_rcu_sched(). */
  117. #define call_rcu call_rcu_sched
  118. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  119. /**
  120. * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
  121. * @head: structure to be used for queueing the RCU updates.
  122. * @func: actual callback function to be invoked after the grace period
  123. *
  124. * The callback function will be invoked some time after a full grace
  125. * period elapses, in other words after all currently executing RCU
  126. * read-side critical sections have completed. call_rcu_bh() assumes
  127. * that the read-side critical sections end on completion of a softirq
  128. * handler. This means that read-side critical sections in process
  129. * context must not be interrupted by softirqs. This interface is to be
  130. * used when most of the read-side critical sections are in softirq context.
  131. * RCU read-side critical sections are delimited by :
  132. * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
  133. * OR
  134. * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
  135. * These may be nested.
  136. *
  137. * See the description of call_rcu() for more detailed information on
  138. * memory ordering guarantees.
  139. */
  140. extern void call_rcu_bh(struct rcu_head *head,
  141. void (*func)(struct rcu_head *head));
  142. /**
  143. * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
  144. * @head: structure to be used for queueing the RCU updates.
  145. * @func: actual callback function to be invoked after the grace period
  146. *
  147. * The callback function will be invoked some time after a full grace
  148. * period elapses, in other words after all currently executing RCU
  149. * read-side critical sections have completed. call_rcu_sched() assumes
  150. * that the read-side critical sections end on enabling of preemption
  151. * or on voluntary preemption.
  152. * RCU read-side critical sections are delimited by :
  153. * - rcu_read_lock_sched() and rcu_read_unlock_sched(),
  154. * OR
  155. * anything that disables preemption.
  156. * These may be nested.
  157. *
  158. * See the description of call_rcu() for more detailed information on
  159. * memory ordering guarantees.
  160. */
  161. extern void call_rcu_sched(struct rcu_head *head,
  162. void (*func)(struct rcu_head *rcu));
  163. extern void synchronize_sched(void);
  164. #ifdef CONFIG_PREEMPT_RCU
  165. extern void __rcu_read_lock(void);
  166. extern void __rcu_read_unlock(void);
  167. extern void rcu_read_unlock_special(struct task_struct *t);
  168. void synchronize_rcu(void);
  169. /*
  170. * Defined as a macro as it is a very low level header included from
  171. * areas that don't even know about current. This gives the rcu_read_lock()
  172. * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
  173. * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
  174. */
  175. #define rcu_preempt_depth() (current->rcu_read_lock_nesting)
  176. #else /* #ifdef CONFIG_PREEMPT_RCU */
  177. static inline void __rcu_read_lock(void)
  178. {
  179. preempt_disable();
  180. }
  181. static inline void __rcu_read_unlock(void)
  182. {
  183. preempt_enable();
  184. }
  185. static inline void synchronize_rcu(void)
  186. {
  187. synchronize_sched();
  188. }
  189. static inline int rcu_preempt_depth(void)
  190. {
  191. return 0;
  192. }
  193. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  194. /* Internal to kernel */
  195. extern void rcu_sched_qs(int cpu);
  196. extern void rcu_bh_qs(int cpu);
  197. extern void rcu_check_callbacks(int cpu, int user);
  198. struct notifier_block;
  199. extern void rcu_idle_enter(void);
  200. extern void rcu_idle_exit(void);
  201. extern void rcu_irq_enter(void);
  202. extern void rcu_irq_exit(void);
  203. #ifdef CONFIG_RCU_USER_QS
  204. extern void rcu_user_enter(void);
  205. extern void rcu_user_exit(void);
  206. extern void rcu_user_enter_after_irq(void);
  207. extern void rcu_user_exit_after_irq(void);
  208. #else
  209. static inline void rcu_user_enter(void) { }
  210. static inline void rcu_user_exit(void) { }
  211. static inline void rcu_user_enter_after_irq(void) { }
  212. static inline void rcu_user_exit_after_irq(void) { }
  213. static inline void rcu_user_hooks_switch(struct task_struct *prev,
  214. struct task_struct *next) { }
  215. #endif /* CONFIG_RCU_USER_QS */
  216. extern void exit_rcu(void);
  217. /**
  218. * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
  219. * @a: Code that RCU needs to pay attention to.
  220. *
  221. * RCU, RCU-bh, and RCU-sched read-side critical sections are forbidden
  222. * in the inner idle loop, that is, between the rcu_idle_enter() and
  223. * the rcu_idle_exit() -- RCU will happily ignore any such read-side
  224. * critical sections. However, things like powertop need tracepoints
  225. * in the inner idle loop.
  226. *
  227. * This macro provides the way out: RCU_NONIDLE(do_something_with_RCU())
  228. * will tell RCU that it needs to pay attending, invoke its argument
  229. * (in this example, a call to the do_something_with_RCU() function),
  230. * and then tell RCU to go back to ignoring this CPU. It is permissible
  231. * to nest RCU_NONIDLE() wrappers, but the nesting level is currently
  232. * quite limited. If deeper nesting is required, it will be necessary
  233. * to adjust DYNTICK_TASK_NESTING_VALUE accordingly.
  234. */
  235. #define RCU_NONIDLE(a) \
  236. do { \
  237. rcu_irq_enter(); \
  238. do { a; } while (0); \
  239. rcu_irq_exit(); \
  240. } while (0)
  241. /*
  242. * Infrastructure to implement the synchronize_() primitives in
  243. * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
  244. */
  245. typedef void call_rcu_func_t(struct rcu_head *head,
  246. void (*func)(struct rcu_head *head));
  247. void wait_rcu_gp(call_rcu_func_t crf);
  248. #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
  249. #include <linux/rcutree.h>
  250. #elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
  251. #include <linux/rcutiny.h>
  252. #else
  253. #error "Unknown RCU implementation specified to kernel configuration"
  254. #endif
  255. /*
  256. * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
  257. * initialization and destruction of rcu_head on the stack. rcu_head structures
  258. * allocated dynamically in the heap or defined statically don't need any
  259. * initialization.
  260. */
  261. #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
  262. extern void init_rcu_head_on_stack(struct rcu_head *head);
  263. extern void destroy_rcu_head_on_stack(struct rcu_head *head);
  264. #else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
  265. static inline void init_rcu_head_on_stack(struct rcu_head *head)
  266. {
  267. }
  268. static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
  269. {
  270. }
  271. #endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
  272. #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP)
  273. extern int rcu_is_cpu_idle(void);
  274. #endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP) */
  275. #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
  276. bool rcu_lockdep_current_cpu_online(void);
  277. #else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
  278. static inline bool rcu_lockdep_current_cpu_online(void)
  279. {
  280. return 1;
  281. }
  282. #endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
  283. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  284. static inline void rcu_lock_acquire(struct lockdep_map *map)
  285. {
  286. lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);
  287. }
  288. static inline void rcu_lock_release(struct lockdep_map *map)
  289. {
  290. lock_release(map, 1, _THIS_IP_);
  291. }
  292. extern struct lockdep_map rcu_lock_map;
  293. extern struct lockdep_map rcu_bh_lock_map;
  294. extern struct lockdep_map rcu_sched_lock_map;
  295. extern int debug_lockdep_rcu_enabled(void);
  296. /**
  297. * rcu_read_lock_held() - might we be in RCU read-side critical section?
  298. *
  299. * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
  300. * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
  301. * this assumes we are in an RCU read-side critical section unless it can
  302. * prove otherwise. This is useful for debug checks in functions that
  303. * require that they be called within an RCU read-side critical section.
  304. *
  305. * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
  306. * and while lockdep is disabled.
  307. *
  308. * Note that rcu_read_lock() and the matching rcu_read_unlock() must
  309. * occur in the same context, for example, it is illegal to invoke
  310. * rcu_read_unlock() in process context if the matching rcu_read_lock()
  311. * was invoked from within an irq handler.
  312. *
  313. * Note that rcu_read_lock() is disallowed if the CPU is either idle or
  314. * offline from an RCU perspective, so check for those as well.
  315. */
  316. static inline int rcu_read_lock_held(void)
  317. {
  318. if (!debug_lockdep_rcu_enabled())
  319. return 1;
  320. if (rcu_is_cpu_idle())
  321. return 0;
  322. if (!rcu_lockdep_current_cpu_online())
  323. return 0;
  324. return lock_is_held(&rcu_lock_map);
  325. }
  326. /*
  327. * rcu_read_lock_bh_held() is defined out of line to avoid #include-file
  328. * hell.
  329. */
  330. extern int rcu_read_lock_bh_held(void);
  331. /**
  332. * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
  333. *
  334. * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
  335. * RCU-sched read-side critical section. In absence of
  336. * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
  337. * critical section unless it can prove otherwise. Note that disabling
  338. * of preemption (including disabling irqs) counts as an RCU-sched
  339. * read-side critical section. This is useful for debug checks in functions
  340. * that required that they be called within an RCU-sched read-side
  341. * critical section.
  342. *
  343. * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
  344. * and while lockdep is disabled.
  345. *
  346. * Note that if the CPU is in the idle loop from an RCU point of
  347. * view (ie: that we are in the section between rcu_idle_enter() and
  348. * rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU
  349. * did an rcu_read_lock(). The reason for this is that RCU ignores CPUs
  350. * that are in such a section, considering these as in extended quiescent
  351. * state, so such a CPU is effectively never in an RCU read-side critical
  352. * section regardless of what RCU primitives it invokes. This state of
  353. * affairs is required --- we need to keep an RCU-free window in idle
  354. * where the CPU may possibly enter into low power mode. This way we can
  355. * notice an extended quiescent state to other CPUs that started a grace
  356. * period. Otherwise we would delay any grace period as long as we run in
  357. * the idle task.
  358. *
  359. * Similarly, we avoid claiming an SRCU read lock held if the current
  360. * CPU is offline.
  361. */
  362. #ifdef CONFIG_PREEMPT_COUNT
  363. static inline int rcu_read_lock_sched_held(void)
  364. {
  365. int lockdep_opinion = 0;
  366. if (!debug_lockdep_rcu_enabled())
  367. return 1;
  368. if (rcu_is_cpu_idle())
  369. return 0;
  370. if (!rcu_lockdep_current_cpu_online())
  371. return 0;
  372. if (debug_locks)
  373. lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
  374. return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
  375. }
  376. #else /* #ifdef CONFIG_PREEMPT_COUNT */
  377. static inline int rcu_read_lock_sched_held(void)
  378. {
  379. return 1;
  380. }
  381. #endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
  382. #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
  383. # define rcu_lock_acquire(a) do { } while (0)
  384. # define rcu_lock_release(a) do { } while (0)
  385. static inline int rcu_read_lock_held(void)
  386. {
  387. return 1;
  388. }
  389. static inline int rcu_read_lock_bh_held(void)
  390. {
  391. return 1;
  392. }
  393. #ifdef CONFIG_PREEMPT_COUNT
  394. static inline int rcu_read_lock_sched_held(void)
  395. {
  396. return preempt_count() != 0 || irqs_disabled();
  397. }
  398. #else /* #ifdef CONFIG_PREEMPT_COUNT */
  399. static inline int rcu_read_lock_sched_held(void)
  400. {
  401. return 1;
  402. }
  403. #endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
  404. #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
  405. #ifdef CONFIG_PROVE_RCU
  406. extern int rcu_my_thread_group_empty(void);
  407. /**
  408. * rcu_lockdep_assert - emit lockdep splat if specified condition not met
  409. * @c: condition to check
  410. * @s: informative message
  411. */
  412. #define rcu_lockdep_assert(c, s) \
  413. do { \
  414. static bool __section(.data.unlikely) __warned; \
  415. if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
  416. __warned = true; \
  417. lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
  418. } \
  419. } while (0)
  420. #if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
  421. static inline void rcu_preempt_sleep_check(void)
  422. {
  423. rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),
  424. "Illegal context switch in RCU read-side critical section");
  425. }
  426. #else /* #ifdef CONFIG_PROVE_RCU */
  427. static inline void rcu_preempt_sleep_check(void)
  428. {
  429. }
  430. #endif /* #else #ifdef CONFIG_PROVE_RCU */
  431. #define rcu_sleep_check() \
  432. do { \
  433. rcu_preempt_sleep_check(); \
  434. rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
  435. "Illegal context switch in RCU-bh" \
  436. " read-side critical section"); \
  437. rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
  438. "Illegal context switch in RCU-sched"\
  439. " read-side critical section"); \
  440. } while (0)
  441. #else /* #ifdef CONFIG_PROVE_RCU */
  442. #define rcu_lockdep_assert(c, s) do { } while (0)
  443. #define rcu_sleep_check() do { } while (0)
  444. #endif /* #else #ifdef CONFIG_PROVE_RCU */
  445. /*
  446. * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
  447. * and rcu_assign_pointer(). Some of these could be folded into their
  448. * callers, but they are left separate in order to ease introduction of
  449. * multiple flavors of pointers to match the multiple flavors of RCU
  450. * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
  451. * the future.
  452. */
  453. #ifdef __CHECKER__
  454. #define rcu_dereference_sparse(p, space) \
  455. ((void)(((typeof(*p) space *)p) == p))
  456. #else /* #ifdef __CHECKER__ */
  457. #define rcu_dereference_sparse(p, space)
  458. #endif /* #else #ifdef __CHECKER__ */
  459. #define __rcu_access_pointer(p, space) \
  460. ({ \
  461. typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
  462. rcu_dereference_sparse(p, space); \
  463. ((typeof(*p) __force __kernel *)(_________p1)); \
  464. })
  465. #define __rcu_dereference_check(p, c, space) \
  466. ({ \
  467. typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
  468. rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
  469. " usage"); \
  470. rcu_dereference_sparse(p, space); \
  471. smp_read_barrier_depends(); \
  472. ((typeof(*p) __force __kernel *)(_________p1)); \
  473. })
  474. #define __rcu_dereference_protected(p, c, space) \
  475. ({ \
  476. rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
  477. " usage"); \
  478. rcu_dereference_sparse(p, space); \
  479. ((typeof(*p) __force __kernel *)(p)); \
  480. })
  481. #define __rcu_access_index(p, space) \
  482. ({ \
  483. typeof(p) _________p1 = ACCESS_ONCE(p); \
  484. rcu_dereference_sparse(p, space); \
  485. (_________p1); \
  486. })
  487. #define __rcu_dereference_index_check(p, c) \
  488. ({ \
  489. typeof(p) _________p1 = ACCESS_ONCE(p); \
  490. rcu_lockdep_assert(c, \
  491. "suspicious rcu_dereference_index_check()" \
  492. " usage"); \
  493. smp_read_barrier_depends(); \
  494. (_________p1); \
  495. })
  496. #define __rcu_assign_pointer(p, v, space) \
  497. do { \
  498. smp_wmb(); \
  499. (p) = (typeof(*v) __force space *)(v); \
  500. } while (0)
  501. /**
  502. * rcu_access_pointer() - fetch RCU pointer with no dereferencing
  503. * @p: The pointer to read
  504. *
  505. * Return the value of the specified RCU-protected pointer, but omit the
  506. * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
  507. * when the value of this pointer is accessed, but the pointer is not
  508. * dereferenced, for example, when testing an RCU-protected pointer against
  509. * NULL. Although rcu_access_pointer() may also be used in cases where
  510. * update-side locks prevent the value of the pointer from changing, you
  511. * should instead use rcu_dereference_protected() for this use case.
  512. *
  513. * It is also permissible to use rcu_access_pointer() when read-side
  514. * access to the pointer was removed at least one grace period ago, as
  515. * is the case in the context of the RCU callback that is freeing up
  516. * the data, or after a synchronize_rcu() returns. This can be useful
  517. * when tearing down multi-linked structures after a grace period
  518. * has elapsed.
  519. */
  520. #define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
  521. /**
  522. * rcu_dereference_check() - rcu_dereference with debug checking
  523. * @p: The pointer to read, prior to dereferencing
  524. * @c: The conditions under which the dereference will take place
  525. *
  526. * Do an rcu_dereference(), but check that the conditions under which the
  527. * dereference will take place are correct. Typically the conditions
  528. * indicate the various locking conditions that should be held at that
  529. * point. The check should return true if the conditions are satisfied.
  530. * An implicit check for being in an RCU read-side critical section
  531. * (rcu_read_lock()) is included.
  532. *
  533. * For example:
  534. *
  535. * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
  536. *
  537. * could be used to indicate to lockdep that foo->bar may only be dereferenced
  538. * if either rcu_read_lock() is held, or that the lock required to replace
  539. * the bar struct at foo->bar is held.
  540. *
  541. * Note that the list of conditions may also include indications of when a lock
  542. * need not be held, for example during initialisation or destruction of the
  543. * target struct:
  544. *
  545. * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
  546. * atomic_read(&foo->usage) == 0);
  547. *
  548. * Inserts memory barriers on architectures that require them
  549. * (currently only the Alpha), prevents the compiler from refetching
  550. * (and from merging fetches), and, more importantly, documents exactly
  551. * which pointers are protected by RCU and checks that the pointer is
  552. * annotated as __rcu.
  553. */
  554. #define rcu_dereference_check(p, c) \
  555. __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
  556. /**
  557. * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
  558. * @p: The pointer to read, prior to dereferencing
  559. * @c: The conditions under which the dereference will take place
  560. *
  561. * This is the RCU-bh counterpart to rcu_dereference_check().
  562. */
  563. #define rcu_dereference_bh_check(p, c) \
  564. __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
  565. /**
  566. * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
  567. * @p: The pointer to read, prior to dereferencing
  568. * @c: The conditions under which the dereference will take place
  569. *
  570. * This is the RCU-sched counterpart to rcu_dereference_check().
  571. */
  572. #define rcu_dereference_sched_check(p, c) \
  573. __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
  574. __rcu)
  575. #define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
  576. /**
  577. * rcu_access_index() - fetch RCU index with no dereferencing
  578. * @p: The index to read
  579. *
  580. * Return the value of the specified RCU-protected index, but omit the
  581. * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
  582. * when the value of this index is accessed, but the index is not
  583. * dereferenced, for example, when testing an RCU-protected index against
  584. * -1. Although rcu_access_index() may also be used in cases where
  585. * update-side locks prevent the value of the index from changing, you
  586. * should instead use rcu_dereference_index_protected() for this use case.
  587. */
  588. #define rcu_access_index(p) __rcu_access_index((p), __rcu)
  589. /**
  590. * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
  591. * @p: The pointer to read, prior to dereferencing
  592. * @c: The conditions under which the dereference will take place
  593. *
  594. * Similar to rcu_dereference_check(), but omits the sparse checking.
  595. * This allows rcu_dereference_index_check() to be used on integers,
  596. * which can then be used as array indices. Attempting to use
  597. * rcu_dereference_check() on an integer will give compiler warnings
  598. * because the sparse address-space mechanism relies on dereferencing
  599. * the RCU-protected pointer. Dereferencing integers is not something
  600. * that even gcc will put up with.
  601. *
  602. * Note that this function does not implicitly check for RCU read-side
  603. * critical sections. If this function gains lots of uses, it might
  604. * make sense to provide versions for each flavor of RCU, but it does
  605. * not make sense as of early 2010.
  606. */
  607. #define rcu_dereference_index_check(p, c) \
  608. __rcu_dereference_index_check((p), (c))
  609. /**
  610. * rcu_dereference_protected() - fetch RCU pointer when updates prevented
  611. * @p: The pointer to read, prior to dereferencing
  612. * @c: The conditions under which the dereference will take place
  613. *
  614. * Return the value of the specified RCU-protected pointer, but omit
  615. * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
  616. * is useful in cases where update-side locks prevent the value of the
  617. * pointer from changing. Please note that this primitive does -not-
  618. * prevent the compiler from repeating this reference or combining it
  619. * with other references, so it should not be used without protection
  620. * of appropriate locks.
  621. *
  622. * This function is only for update-side use. Using this function
  623. * when protected only by rcu_read_lock() will result in infrequent
  624. * but very ugly failures.
  625. */
  626. #define rcu_dereference_protected(p, c) \
  627. __rcu_dereference_protected((p), (c), __rcu)
  628. /**
  629. * rcu_dereference() - fetch RCU-protected pointer for dereferencing
  630. * @p: The pointer to read, prior to dereferencing
  631. *
  632. * This is a simple wrapper around rcu_dereference_check().
  633. */
  634. #define rcu_dereference(p) rcu_dereference_check(p, 0)
  635. /**
  636. * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
  637. * @p: The pointer to read, prior to dereferencing
  638. *
  639. * Makes rcu_dereference_check() do the dirty work.
  640. */
  641. #define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
  642. /**
  643. * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
  644. * @p: The pointer to read, prior to dereferencing
  645. *
  646. * Makes rcu_dereference_check() do the dirty work.
  647. */
  648. #define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
  649. /**
  650. * rcu_read_lock() - mark the beginning of an RCU read-side critical section
  651. *
  652. * When synchronize_rcu() is invoked on one CPU while other CPUs
  653. * are within RCU read-side critical sections, then the
  654. * synchronize_rcu() is guaranteed to block until after all the other
  655. * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
  656. * on one CPU while other CPUs are within RCU read-side critical
  657. * sections, invocation of the corresponding RCU callback is deferred
  658. * until after the all the other CPUs exit their critical sections.
  659. *
  660. * Note, however, that RCU callbacks are permitted to run concurrently
  661. * with new RCU read-side critical sections. One way that this can happen
  662. * is via the following sequence of events: (1) CPU 0 enters an RCU
  663. * read-side critical section, (2) CPU 1 invokes call_rcu() to register
  664. * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
  665. * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
  666. * callback is invoked. This is legal, because the RCU read-side critical
  667. * section that was running concurrently with the call_rcu() (and which
  668. * therefore might be referencing something that the corresponding RCU
  669. * callback would free up) has completed before the corresponding
  670. * RCU callback is invoked.
  671. *
  672. * RCU read-side critical sections may be nested. Any deferred actions
  673. * will be deferred until the outermost RCU read-side critical section
  674. * completes.
  675. *
  676. * You can avoid reading and understanding the next paragraph by
  677. * following this rule: don't put anything in an rcu_read_lock() RCU
  678. * read-side critical section that would block in a !PREEMPT kernel.
  679. * But if you want the full story, read on!
  680. *
  681. * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
  682. * is illegal to block while in an RCU read-side critical section. In
  683. * preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
  684. * in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
  685. * be preempted, but explicit blocking is illegal. Finally, in preemptible
  686. * RCU implementations in real-time (with -rt patchset) kernel builds,
  687. * RCU read-side critical sections may be preempted and they may also
  688. * block, but only when acquiring spinlocks that are subject to priority
  689. * inheritance.
  690. */
  691. static inline void rcu_read_lock(void)
  692. {
  693. __rcu_read_lock();
  694. __acquire(RCU);
  695. rcu_lock_acquire(&rcu_lock_map);
  696. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  697. "rcu_read_lock() used illegally while idle");
  698. }
  699. /*
  700. * So where is rcu_write_lock()? It does not exist, as there is no
  701. * way for writers to lock out RCU readers. This is a feature, not
  702. * a bug -- this property is what provides RCU's performance benefits.
  703. * Of course, writers must coordinate with each other. The normal
  704. * spinlock primitives work well for this, but any other technique may be
  705. * used as well. RCU does not care how the writers keep out of each
  706. * others' way, as long as they do so.
  707. */
  708. /**
  709. * rcu_read_unlock() - marks the end of an RCU read-side critical section.
  710. *
  711. * See rcu_read_lock() for more information.
  712. */
  713. static inline void rcu_read_unlock(void)
  714. {
  715. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  716. "rcu_read_unlock() used illegally while idle");
  717. rcu_lock_release(&rcu_lock_map);
  718. __release(RCU);
  719. __rcu_read_unlock();
  720. }
  721. /**
  722. * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
  723. *
  724. * This is equivalent of rcu_read_lock(), but to be used when updates
  725. * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
  726. * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
  727. * softirq handler to be a quiescent state, a process in RCU read-side
  728. * critical section must be protected by disabling softirqs. Read-side
  729. * critical sections in interrupt context can use just rcu_read_lock(),
  730. * though this should at least be commented to avoid confusing people
  731. * reading the code.
  732. *
  733. * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
  734. * must occur in the same context, for example, it is illegal to invoke
  735. * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
  736. * was invoked from some other task.
  737. */
  738. static inline void rcu_read_lock_bh(void)
  739. {
  740. local_bh_disable();
  741. __acquire(RCU_BH);
  742. rcu_lock_acquire(&rcu_bh_lock_map);
  743. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  744. "rcu_read_lock_bh() used illegally while idle");
  745. }
  746. /*
  747. * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
  748. *
  749. * See rcu_read_lock_bh() for more information.
  750. */
  751. static inline void rcu_read_unlock_bh(void)
  752. {
  753. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  754. "rcu_read_unlock_bh() used illegally while idle");
  755. rcu_lock_release(&rcu_bh_lock_map);
  756. __release(RCU_BH);
  757. local_bh_enable();
  758. }
  759. /**
  760. * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
  761. *
  762. * This is equivalent of rcu_read_lock(), but to be used when updates
  763. * are being done using call_rcu_sched() or synchronize_rcu_sched().
  764. * Read-side critical sections can also be introduced by anything that
  765. * disables preemption, including local_irq_disable() and friends.
  766. *
  767. * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
  768. * must occur in the same context, for example, it is illegal to invoke
  769. * rcu_read_unlock_sched() from process context if the matching
  770. * rcu_read_lock_sched() was invoked from an NMI handler.
  771. */
  772. static inline void rcu_read_lock_sched(void)
  773. {
  774. preempt_disable();
  775. __acquire(RCU_SCHED);
  776. rcu_lock_acquire(&rcu_sched_lock_map);
  777. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  778. "rcu_read_lock_sched() used illegally while idle");
  779. }
  780. /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
  781. static inline notrace void rcu_read_lock_sched_notrace(void)
  782. {
  783. preempt_disable_notrace();
  784. __acquire(RCU_SCHED);
  785. }
  786. /*
  787. * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
  788. *
  789. * See rcu_read_lock_sched for more information.
  790. */
  791. static inline void rcu_read_unlock_sched(void)
  792. {
  793. rcu_lockdep_assert(!rcu_is_cpu_idle(),
  794. "rcu_read_unlock_sched() used illegally while idle");
  795. rcu_lock_release(&rcu_sched_lock_map);
  796. __release(RCU_SCHED);
  797. preempt_enable();
  798. }
  799. /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
  800. static inline notrace void rcu_read_unlock_sched_notrace(void)
  801. {
  802. __release(RCU_SCHED);
  803. preempt_enable_notrace();
  804. }
  805. /**
  806. * rcu_assign_pointer() - assign to RCU-protected pointer
  807. * @p: pointer to assign to
  808. * @v: value to assign (publish)
  809. *
  810. * Assigns the specified value to the specified RCU-protected
  811. * pointer, ensuring that any concurrent RCU readers will see
  812. * any prior initialization.
  813. *
  814. * Inserts memory barriers on architectures that require them
  815. * (which is most of them), and also prevents the compiler from
  816. * reordering the code that initializes the structure after the pointer
  817. * assignment. More importantly, this call documents which pointers
  818. * will be dereferenced by RCU read-side code.
  819. *
  820. * In some special cases, you may use RCU_INIT_POINTER() instead
  821. * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
  822. * to the fact that it does not constrain either the CPU or the compiler.
  823. * That said, using RCU_INIT_POINTER() when you should have used
  824. * rcu_assign_pointer() is a very bad thing that results in
  825. * impossible-to-diagnose memory corruption. So please be careful.
  826. * See the RCU_INIT_POINTER() comment header for details.
  827. */
  828. #define rcu_assign_pointer(p, v) \
  829. __rcu_assign_pointer((p), (v), __rcu)
  830. /**
  831. * RCU_INIT_POINTER() - initialize an RCU protected pointer
  832. *
  833. * Initialize an RCU-protected pointer in special cases where readers
  834. * do not need ordering constraints on the CPU or the compiler. These
  835. * special cases are:
  836. *
  837. * 1. This use of RCU_INIT_POINTER() is NULLing out the pointer -or-
  838. * 2. The caller has taken whatever steps are required to prevent
  839. * RCU readers from concurrently accessing this pointer -or-
  840. * 3. The referenced data structure has already been exposed to
  841. * readers either at compile time or via rcu_assign_pointer() -and-
  842. * a. You have not made -any- reader-visible changes to
  843. * this structure since then -or-
  844. * b. It is OK for readers accessing this structure from its
  845. * new location to see the old state of the structure. (For
  846. * example, the changes were to statistical counters or to
  847. * other state where exact synchronization is not required.)
  848. *
  849. * Failure to follow these rules governing use of RCU_INIT_POINTER() will
  850. * result in impossible-to-diagnose memory corruption. As in the structures
  851. * will look OK in crash dumps, but any concurrent RCU readers might
  852. * see pre-initialized values of the referenced data structure. So
  853. * please be very careful how you use RCU_INIT_POINTER()!!!
  854. *
  855. * If you are creating an RCU-protected linked structure that is accessed
  856. * by a single external-to-structure RCU-protected pointer, then you may
  857. * use RCU_INIT_POINTER() to initialize the internal RCU-protected
  858. * pointers, but you must use rcu_assign_pointer() to initialize the
  859. * external-to-structure pointer -after- you have completely initialized
  860. * the reader-accessible portions of the linked structure.
  861. */
  862. #define RCU_INIT_POINTER(p, v) \
  863. do { \
  864. p = (typeof(*v) __force __rcu *)(v); \
  865. } while (0)
  866. /**
  867. * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
  868. *
  869. * GCC-style initialization for an RCU-protected pointer in a structure field.
  870. */
  871. #define RCU_POINTER_INITIALIZER(p, v) \
  872. .p = (typeof(*v) __force __rcu *)(v)
  873. /*
  874. * Does the specified offset indicate that the corresponding rcu_head
  875. * structure can be handled by kfree_rcu()?
  876. */
  877. #define __is_kfree_rcu_offset(offset) ((offset) < 4096)
  878. /*
  879. * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
  880. */
  881. #define __kfree_rcu(head, offset) \
  882. do { \
  883. BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
  884. kfree_call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
  885. } while (0)
  886. /**
  887. * kfree_rcu() - kfree an object after a grace period.
  888. * @ptr: pointer to kfree
  889. * @rcu_head: the name of the struct rcu_head within the type of @ptr.
  890. *
  891. * Many rcu callbacks functions just call kfree() on the base structure.
  892. * These functions are trivial, but their size adds up, and furthermore
  893. * when they are used in a kernel module, that module must invoke the
  894. * high-latency rcu_barrier() function at module-unload time.
  895. *
  896. * The kfree_rcu() function handles this issue. Rather than encoding a
  897. * function address in the embedded rcu_head structure, kfree_rcu() instead
  898. * encodes the offset of the rcu_head structure within the base structure.
  899. * Because the functions are not allowed in the low-order 4096 bytes of
  900. * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
  901. * If the offset is larger than 4095 bytes, a compile-time error will
  902. * be generated in __kfree_rcu(). If this error is triggered, you can
  903. * either fall back to use of call_rcu() or rearrange the structure to
  904. * position the rcu_head structure into the first 4096 bytes.
  905. *
  906. * Note that the allowable offset might decrease in the future, for example,
  907. * to allow something like kmem_cache_free_rcu().
  908. *
  909. * The BUILD_BUG_ON check must not involve any function calls, hence the
  910. * checks are done in macros here.
  911. */
  912. #define kfree_rcu(ptr, rcu_head) \
  913. __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
  914. #endif /* __LINUX_RCUPDATE_H */