|
@@ -476,8 +476,11 @@ static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
|
|
|
static void migrate_page_add(struct page *page, struct list_head *pagelist,
|
|
|
unsigned long flags);
|
|
|
|
|
|
-/* Scan through pages checking if pages follow certain conditions. */
|
|
|
-static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
|
+/*
|
|
|
+ * Scan through pages checking if pages follow certain conditions,
|
|
|
+ * and move them to the pagelist if they do.
|
|
|
+ */
|
|
|
+static int queue_pages_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
|
unsigned long addr, unsigned long end,
|
|
|
const nodemask_t *nodes, unsigned long flags,
|
|
|
void *private)
|
|
@@ -515,8 +518,8 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
|
return addr != end;
|
|
|
}
|
|
|
|
|
|
-static void check_hugetlb_pmd_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
|
- const nodemask_t *nodes, unsigned long flags,
|
|
|
+static void queue_pages_hugetlb_pmd_range(struct vm_area_struct *vma,
|
|
|
+ pmd_t *pmd, const nodemask_t *nodes, unsigned long flags,
|
|
|
void *private)
|
|
|
{
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
@@ -539,7 +542,7 @@ unlock:
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
-static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
|
|
|
+static inline int queue_pages_pmd_range(struct vm_area_struct *vma, pud_t *pud,
|
|
|
unsigned long addr, unsigned long end,
|
|
|
const nodemask_t *nodes, unsigned long flags,
|
|
|
void *private)
|
|
@@ -553,21 +556,21 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
|
|
|
if (!pmd_present(*pmd))
|
|
|
continue;
|
|
|
if (pmd_huge(*pmd) && is_vm_hugetlb_page(vma)) {
|
|
|
- check_hugetlb_pmd_range(vma, pmd, nodes,
|
|
|
+ queue_pages_hugetlb_pmd_range(vma, pmd, nodes,
|
|
|
flags, private);
|
|
|
continue;
|
|
|
}
|
|
|
split_huge_page_pmd(vma, addr, pmd);
|
|
|
if (pmd_none_or_trans_huge_or_clear_bad(pmd))
|
|
|
continue;
|
|
|
- if (check_pte_range(vma, pmd, addr, next, nodes,
|
|
|
+ if (queue_pages_pte_range(vma, pmd, addr, next, nodes,
|
|
|
flags, private))
|
|
|
return -EIO;
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
|
|
|
+static inline int queue_pages_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
|
|
|
unsigned long addr, unsigned long end,
|
|
|
const nodemask_t *nodes, unsigned long flags,
|
|
|
void *private)
|
|
@@ -582,14 +585,14 @@ static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
|
|
|
continue;
|
|
|
if (pud_none_or_clear_bad(pud))
|
|
|
continue;
|
|
|
- if (check_pmd_range(vma, pud, addr, next, nodes,
|
|
|
+ if (queue_pages_pmd_range(vma, pud, addr, next, nodes,
|
|
|
flags, private))
|
|
|
return -EIO;
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static inline int check_pgd_range(struct vm_area_struct *vma,
|
|
|
+static inline int queue_pages_pgd_range(struct vm_area_struct *vma,
|
|
|
unsigned long addr, unsigned long end,
|
|
|
const nodemask_t *nodes, unsigned long flags,
|
|
|
void *private)
|
|
@@ -602,7 +605,7 @@ static inline int check_pgd_range(struct vm_area_struct *vma,
|
|
|
next = pgd_addr_end(addr, end);
|
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
|
continue;
|
|
|
- if (check_pud_range(vma, pgd, addr, next, nodes,
|
|
|
+ if (queue_pages_pud_range(vma, pgd, addr, next, nodes,
|
|
|
flags, private))
|
|
|
return -EIO;
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
@@ -640,12 +643,14 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma,
|
|
|
#endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */
|
|
|
|
|
|
/*
|
|
|
- * Check if all pages in a range are on a set of nodes.
|
|
|
- * If pagelist != NULL then isolate pages from the LRU and
|
|
|
- * put them on the pagelist.
|
|
|
+ * Walk through page tables and collect pages to be migrated.
|
|
|
+ *
|
|
|
+ * If pages found in a given range are on a set of nodes (determined by
|
|
|
+ * @nodes and @flags,) it's isolated and queued to the pagelist which is
|
|
|
+ * passed via @private.)
|
|
|
*/
|
|
|
static struct vm_area_struct *
|
|
|
-check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
|
|
|
+queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
|
|
|
const nodemask_t *nodes, unsigned long flags, void *private)
|
|
|
{
|
|
|
int err;
|
|
@@ -680,7 +685,7 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
|
|
|
((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
|
|
|
vma_migratable(vma))) {
|
|
|
|
|
|
- err = check_pgd_range(vma, start, endvma, nodes,
|
|
|
+ err = queue_pages_pgd_range(vma, start, endvma, nodes,
|
|
|
flags, private);
|
|
|
if (err) {
|
|
|
first = ERR_PTR(err);
|
|
@@ -1050,7 +1055,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
|
|
|
* space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
|
|
|
*/
|
|
|
VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
|
|
|
- check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
|
|
|
+ queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
|
|
|
flags | MPOL_MF_DISCONTIG_OK, &pagelist);
|
|
|
|
|
|
if (!list_empty(&pagelist)) {
|
|
@@ -1288,7 +1293,7 @@ static long do_mbind(unsigned long start, unsigned long len,
|
|
|
if (err)
|
|
|
goto mpol_out;
|
|
|
|
|
|
- vma = check_range(mm, start, end, nmask,
|
|
|
+ vma = queue_pages_range(mm, start, end, nmask,
|
|
|
flags | MPOL_MF_INVERT, &pagelist);
|
|
|
|
|
|
err = PTR_ERR(vma); /* maybe ... */
|