|
@@ -940,6 +940,34 @@ out:
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * requeue_futex() - Requeue a futex_q from one hb to another
|
|
|
+ * @q: the futex_q to requeue
|
|
|
+ * @hb1: the source hash_bucket
|
|
|
+ * @hb2: the target hash_bucket
|
|
|
+ * @key2: the new key for the requeued futex_q
|
|
|
+ */
|
|
|
+static inline
|
|
|
+void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
|
|
|
+ struct futex_hash_bucket *hb2, union futex_key *key2)
|
|
|
+{
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If key1 and key2 hash to the same bucket, no need to
|
|
|
+ * requeue.
|
|
|
+ */
|
|
|
+ if (likely(&hb1->chain != &hb2->chain)) {
|
|
|
+ plist_del(&q->list, &hb1->chain);
|
|
|
+ plist_add(&q->list, &hb2->chain);
|
|
|
+ q->lock_ptr = &hb2->lock;
|
|
|
+#ifdef CONFIG_DEBUG_PI_LIST
|
|
|
+ q->list.plist.lock = &hb2->lock;
|
|
|
+#endif
|
|
|
+ }
|
|
|
+ get_futex_key_refs(key2);
|
|
|
+ q->key = *key2;
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Requeue all waiters hashed on one physical page to another
|
|
|
* physical page.
|
|
@@ -999,20 +1027,7 @@ retry_private:
|
|
|
if (++ret <= nr_wake) {
|
|
|
wake_futex(this);
|
|
|
} else {
|
|
|
- /*
|
|
|
- * If key1 and key2 hash to the same bucket, no need to
|
|
|
- * requeue.
|
|
|
- */
|
|
|
- if (likely(head1 != &hb2->chain)) {
|
|
|
- plist_del(&this->list, &hb1->chain);
|
|
|
- plist_add(&this->list, &hb2->chain);
|
|
|
- this->lock_ptr = &hb2->lock;
|
|
|
-#ifdef CONFIG_DEBUG_PI_LIST
|
|
|
- this->list.plist.lock = &hb2->lock;
|
|
|
-#endif
|
|
|
- }
|
|
|
- this->key = key2;
|
|
|
- get_futex_key_refs(&key2);
|
|
|
+ requeue_futex(this, hb1, hb2, &key2);
|
|
|
drop_count++;
|
|
|
|
|
|
if (ret - nr_wake >= nr_requeue)
|