|
@@ -78,14 +78,120 @@ extern void rcu_init(void);
|
|
|
} while (0)
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
+
|
|
|
extern struct lockdep_map rcu_lock_map;
|
|
|
-# define rcu_read_acquire() \
|
|
|
- lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
+# define rcu_read_acquire() \
|
|
|
+ lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
|
|
|
-#else
|
|
|
-# define rcu_read_acquire() do { } while (0)
|
|
|
-# define rcu_read_release() do { } while (0)
|
|
|
-#endif
|
|
|
+
|
|
|
+extern struct lockdep_map rcu_bh_lock_map;
|
|
|
+# define rcu_read_acquire_bh() \
|
|
|
+ lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
+# define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_)
|
|
|
+
|
|
|
+extern struct lockdep_map rcu_sched_lock_map;
|
|
|
+# define rcu_read_acquire_sched() \
|
|
|
+ lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
+# define rcu_read_release_sched() \
|
|
|
+ lock_release(&rcu_sched_lock_map, 1, _THIS_IP_)
|
|
|
+
|
|
|
+/**
|
|
|
+ * rcu_read_lock_held - might we be in RCU read-side critical section?
|
|
|
+ *
|
|
|
+ * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in
|
|
|
+ * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
|
+ * this assumes we are in an RCU read-side critical section unless it can
|
|
|
+ * prove otherwise.
|
|
|
+ */
|
|
|
+static inline int rcu_read_lock_held(void)
|
|
|
+{
|
|
|
+ if (debug_locks)
|
|
|
+ return lock_is_held(&rcu_lock_map);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section?
|
|
|
+ *
|
|
|
+ * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in
|
|
|
+ * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
|
+ * this assumes we are in an RCU-bh read-side critical section unless it can
|
|
|
+ * prove otherwise.
|
|
|
+ */
|
|
|
+static inline int rcu_read_lock_bh_held(void)
|
|
|
+{
|
|
|
+ if (debug_locks)
|
|
|
+ return lock_is_held(&rcu_bh_lock_map);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section?
|
|
|
+ *
|
|
|
+ * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in an
|
|
|
+ * RCU-sched read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
|
+ * this assumes we are in an RCU-sched read-side critical section unless it
|
|
|
+ * can prove otherwise. Note that disabling of preemption (including
|
|
|
+ * disabling irqs) counts as an RCU-sched read-side critical section.
|
|
|
+ */
|
|
|
+static inline int rcu_read_lock_sched_held(void)
|
|
|
+{
|
|
|
+ int lockdep_opinion = 0;
|
|
|
+
|
|
|
+ if (debug_locks)
|
|
|
+ lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
|
|
|
+ return lockdep_opinion || preempt_count() != 0;
|
|
|
+}
|
|
|
+
|
|
|
+#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
+
|
|
|
+# define rcu_read_acquire() do { } while (0)
|
|
|
+# define rcu_read_release() do { } while (0)
|
|
|
+# define rcu_read_acquire_bh() do { } while (0)
|
|
|
+# define rcu_read_release_bh() do { } while (0)
|
|
|
+# define rcu_read_acquire_sched() do { } while (0)
|
|
|
+# define rcu_read_release_sched() do { } while (0)
|
|
|
+
|
|
|
+static inline int rcu_read_lock_held(void)
|
|
|
+{
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static inline int rcu_read_lock_bh_held(void)
|
|
|
+{
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static inline int rcu_read_lock_sched_held(void)
|
|
|
+{
|
|
|
+ return preempt_count() != 0;
|
|
|
+}
|
|
|
+
|
|
|
+#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
+
|
|
|
+#ifdef CONFIG_PROVE_RCU
|
|
|
+
|
|
|
+/**
|
|
|
+ * rcu_dereference_check - rcu_dereference with debug checking
|
|
|
+ *
|
|
|
+ * Do an rcu_dereference(), but check that the context is correct.
|
|
|
+ * For example, rcu_dereference_check(gp, rcu_read_lock_held()) to
|
|
|
+ * ensure that the rcu_dereference_check() executes within an RCU
|
|
|
+ * read-side critical section. It is also possible to check for
|
|
|
+ * locks being held, for example, by using lockdep_is_held().
|
|
|
+ */
|
|
|
+#define rcu_dereference_check(p, c) \
|
|
|
+ ({ \
|
|
|
+ if (debug_locks) \
|
|
|
+ WARN_ON_ONCE(!(c)); \
|
|
|
+ rcu_dereference(p); \
|
|
|
+ })
|
|
|
+
|
|
|
+#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
+
|
|
|
+#define rcu_dereference_check(p, c) rcu_dereference(p)
|
|
|
+
|
|
|
+#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
/**
|
|
|
* rcu_read_lock - mark the beginning of an RCU read-side critical section.
|
|
@@ -160,7 +266,7 @@ static inline void rcu_read_lock_bh(void)
|
|
|
{
|
|
|
__rcu_read_lock_bh();
|
|
|
__acquire(RCU_BH);
|
|
|
- rcu_read_acquire();
|
|
|
+ rcu_read_acquire_bh();
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -170,7 +276,7 @@ static inline void rcu_read_lock_bh(void)
|
|
|
*/
|
|
|
static inline void rcu_read_unlock_bh(void)
|
|
|
{
|
|
|
- rcu_read_release();
|
|
|
+ rcu_read_release_bh();
|
|
|
__release(RCU_BH);
|
|
|
__rcu_read_unlock_bh();
|
|
|
}
|
|
@@ -188,7 +294,7 @@ static inline void rcu_read_lock_sched(void)
|
|
|
{
|
|
|
preempt_disable();
|
|
|
__acquire(RCU_SCHED);
|
|
|
- rcu_read_acquire();
|
|
|
+ rcu_read_acquire_sched();
|
|
|
}
|
|
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
|
@@ -205,7 +311,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void)
|
|
|
*/
|
|
|
static inline void rcu_read_unlock_sched(void)
|
|
|
{
|
|
|
- rcu_read_release();
|
|
|
+ rcu_read_release_sched();
|
|
|
__release(RCU_SCHED);
|
|
|
preempt_enable();
|
|
|
}
|