|
@@ -82,19 +82,6 @@ static void __put_compound_page(struct page *page)
|
|
|
|
|
|
static void put_compound_page(struct page *page)
|
|
|
{
|
|
|
- /*
|
|
|
- * hugetlbfs pages cannot be split from under us. If this is a
|
|
|
- * hugetlbfs page, check refcount on head page and release the page if
|
|
|
- * the refcount becomes zero.
|
|
|
- */
|
|
|
- if (PageHuge(page)) {
|
|
|
- page = compound_head(page);
|
|
|
- if (put_page_testzero(page))
|
|
|
- __put_compound_page(page);
|
|
|
-
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
if (unlikely(PageTail(page))) {
|
|
|
/* __split_huge_page_refcount can run under us */
|
|
|
struct page *page_head = compound_trans_head(page);
|
|
@@ -111,14 +98,31 @@ static void put_compound_page(struct page *page)
|
|
|
* still hot on arches that do not support
|
|
|
* this_cpu_cmpxchg_double().
|
|
|
*/
|
|
|
- if (PageSlab(page_head)) {
|
|
|
- if (PageTail(page)) {
|
|
|
+ if (PageSlab(page_head) || PageHeadHuge(page_head)) {
|
|
|
+ if (likely(PageTail(page))) {
|
|
|
+ /*
|
|
|
+ * __split_huge_page_refcount
|
|
|
+ * cannot race here.
|
|
|
+ */
|
|
|
+ VM_BUG_ON(!PageHead(page_head));
|
|
|
+ atomic_dec(&page->_mapcount);
|
|
|
if (put_page_testzero(page_head))
|
|
|
VM_BUG_ON(1);
|
|
|
-
|
|
|
- atomic_dec(&page->_mapcount);
|
|
|
- goto skip_lock_tail;
|
|
|
+ if (put_page_testzero(page_head))
|
|
|
+ __put_compound_page(page_head);
|
|
|
+ return;
|
|
|
} else
|
|
|
+ /*
|
|
|
+ * __split_huge_page_refcount
|
|
|
+ * run before us, "page" was a
|
|
|
+ * THP tail. The split
|
|
|
+ * page_head has been freed
|
|
|
+ * and reallocated as slab or
|
|
|
+ * hugetlbfs page of smaller
|
|
|
+ * order (only possible if
|
|
|
+ * reallocated as slab on
|
|
|
+ * x86).
|
|
|
+ */
|
|
|
goto skip_lock;
|
|
|
}
|
|
|
/*
|
|
@@ -132,8 +136,27 @@ static void put_compound_page(struct page *page)
|
|
|
/* __split_huge_page_refcount run before us */
|
|
|
compound_unlock_irqrestore(page_head, flags);
|
|
|
skip_lock:
|
|
|
- if (put_page_testzero(page_head))
|
|
|
- __put_single_page(page_head);
|
|
|
+ if (put_page_testzero(page_head)) {
|
|
|
+ /*
|
|
|
+ * The head page may have been
|
|
|
+ * freed and reallocated as a
|
|
|
+ * compound page of smaller
|
|
|
+ * order and then freed again.
|
|
|
+ * All we know is that it
|
|
|
+ * cannot have become: a THP
|
|
|
+ * page, a compound page of
|
|
|
+ * higher order, a tail page.
|
|
|
+ * That is because we still
|
|
|
+ * hold the refcount of the
|
|
|
+ * split THP tail and
|
|
|
+ * page_head was the THP head
|
|
|
+ * before the split.
|
|
|
+ */
|
|
|
+ if (PageHead(page_head))
|
|
|
+ __put_compound_page(page_head);
|
|
|
+ else
|
|
|
+ __put_single_page(page_head);
|
|
|
+ }
|
|
|
out_put_single:
|
|
|
if (put_page_testzero(page))
|
|
|
__put_single_page(page);
|
|
@@ -155,7 +178,6 @@ out_put_single:
|
|
|
VM_BUG_ON(atomic_read(&page->_count) != 0);
|
|
|
compound_unlock_irqrestore(page_head, flags);
|
|
|
|
|
|
-skip_lock_tail:
|
|
|
if (put_page_testzero(page_head)) {
|
|
|
if (PageHead(page_head))
|
|
|
__put_compound_page(page_head);
|
|
@@ -198,51 +220,52 @@ bool __get_page_tail(struct page *page)
|
|
|
* proper PT lock that already serializes against
|
|
|
* split_huge_page().
|
|
|
*/
|
|
|
+ unsigned long flags;
|
|
|
bool got = false;
|
|
|
- struct page *page_head;
|
|
|
-
|
|
|
- /*
|
|
|
- * If this is a hugetlbfs page it cannot be split under us. Simply
|
|
|
- * increment refcount for the head page.
|
|
|
- */
|
|
|
- if (PageHuge(page)) {
|
|
|
- page_head = compound_head(page);
|
|
|
- atomic_inc(&page_head->_count);
|
|
|
- got = true;
|
|
|
- } else {
|
|
|
- unsigned long flags;
|
|
|
+ struct page *page_head = compound_trans_head(page);
|
|
|
|
|
|
- page_head = compound_trans_head(page);
|
|
|
- if (likely(page != page_head &&
|
|
|
- get_page_unless_zero(page_head))) {
|
|
|
-
|
|
|
- /* Ref to put_compound_page() comment. */
|
|
|
- if (PageSlab(page_head)) {
|
|
|
- if (likely(PageTail(page))) {
|
|
|
- __get_page_tail_foll(page, false);
|
|
|
- return true;
|
|
|
- } else {
|
|
|
- put_page(page_head);
|
|
|
- return false;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /*
|
|
|
- * page_head wasn't a dangling pointer but it
|
|
|
- * may not be a head page anymore by the time
|
|
|
- * we obtain the lock. That is ok as long as it
|
|
|
- * can't be freed from under us.
|
|
|
- */
|
|
|
- flags = compound_lock_irqsave(page_head);
|
|
|
- /* here __split_huge_page_refcount won't run anymore */
|
|
|
+ if (likely(page != page_head && get_page_unless_zero(page_head))) {
|
|
|
+ /* Ref to put_compound_page() comment. */
|
|
|
+ if (PageSlab(page_head) || PageHeadHuge(page_head)) {
|
|
|
if (likely(PageTail(page))) {
|
|
|
+ /*
|
|
|
+ * This is a hugetlbfs page or a slab
|
|
|
+ * page. __split_huge_page_refcount
|
|
|
+ * cannot race here.
|
|
|
+ */
|
|
|
+ VM_BUG_ON(!PageHead(page_head));
|
|
|
__get_page_tail_foll(page, false);
|
|
|
- got = true;
|
|
|
- }
|
|
|
- compound_unlock_irqrestore(page_head, flags);
|
|
|
- if (unlikely(!got))
|
|
|
+ return true;
|
|
|
+ } else {
|
|
|
+ /*
|
|
|
+ * __split_huge_page_refcount run
|
|
|
+ * before us, "page" was a THP
|
|
|
+ * tail. The split page_head has been
|
|
|
+ * freed and reallocated as slab or
|
|
|
+ * hugetlbfs page of smaller order
|
|
|
+ * (only possible if reallocated as
|
|
|
+ * slab on x86).
|
|
|
+ */
|
|
|
put_page(page_head);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * page_head wasn't a dangling pointer but it
|
|
|
+ * may not be a head page anymore by the time
|
|
|
+ * we obtain the lock. That is ok as long as it
|
|
|
+ * can't be freed from under us.
|
|
|
+ */
|
|
|
+ flags = compound_lock_irqsave(page_head);
|
|
|
+ /* here __split_huge_page_refcount won't run anymore */
|
|
|
+ if (likely(PageTail(page))) {
|
|
|
+ __get_page_tail_foll(page, false);
|
|
|
+ got = true;
|
|
|
}
|
|
|
+ compound_unlock_irqrestore(page_head, flags);
|
|
|
+ if (unlikely(!got))
|
|
|
+ put_page(page_head);
|
|
|
}
|
|
|
return got;
|
|
|
}
|