|
@@ -333,9 +333,9 @@ static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * This function is called to print an error when a pte in a
|
|
|
- * !VM_UNPAGED region is found pointing to an invalid pfn (which
|
|
|
- * is an error.
|
|
|
+ * This function is called to print an error when a bad pte
|
|
|
+ * is found. For example, we might have a PFN-mapped pte in
|
|
|
+ * a region that doesn't allow it.
|
|
|
*
|
|
|
* The calling function must still handle the error.
|
|
|
*/
|
|
@@ -350,19 +350,56 @@ void print_bad_pte(struct vm_area_struct *vma, pte_t pte, unsigned long vaddr)
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * page_is_anon applies strict checks for an anonymous page belonging to
|
|
|
- * this vma at this address. It is used on VM_UNPAGED vmas, which are
|
|
|
- * usually populated with shared originals (which must not be counted),
|
|
|
- * but occasionally contain private COWed copies (when !VM_SHARED, or
|
|
|
- * perhaps via ptrace when VM_SHARED). An mmap of /dev/mem might window
|
|
|
- * free pages, pages from other processes, or from other parts of this:
|
|
|
- * it's tricky, but try not to be deceived by foreign anonymous pages.
|
|
|
+ * This function gets the "struct page" associated with a pte.
|
|
|
+ *
|
|
|
+ * NOTE! Some mappings do not have "struct pages". A raw PFN mapping
|
|
|
+ * will have each page table entry just pointing to a raw page frame
|
|
|
+ * number, and as far as the VM layer is concerned, those do not have
|
|
|
+ * pages associated with them - even if the PFN might point to memory
|
|
|
+ * that otherwise is perfectly fine and has a "struct page".
|
|
|
+ *
|
|
|
+ * The way we recognize those mappings is through the rules set up
|
|
|
+ * by "remap_pfn_range()": the vma will have the VM_PFNMAP bit set,
|
|
|
+ * and the vm_pgoff will point to the first PFN mapped: thus every
|
|
|
+ * page that is a raw mapping will always honor the rule
|
|
|
+ *
|
|
|
+ * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
|
|
|
+ *
|
|
|
+ * and if that isn't true, the page has been COW'ed (in which case it
|
|
|
+ * _does_ have a "struct page" associated with it even if it is in a
|
|
|
+ * VM_PFNMAP range).
|
|
|
*/
|
|
|
-static inline int page_is_anon(struct page *page,
|
|
|
- struct vm_area_struct *vma, unsigned long addr)
|
|
|
+struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
|
|
|
{
|
|
|
- return page && PageAnon(page) && page_mapped(page) &&
|
|
|
- page_address_in_vma(page, vma) == addr;
|
|
|
+ unsigned long pfn = pte_pfn(pte);
|
|
|
+
|
|
|
+ if (vma->vm_flags & VM_PFNMAP) {
|
|
|
+ unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT;
|
|
|
+ if (pfn == vma->vm_pgoff + off)
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Add some anal sanity checks for now. Eventually,
|
|
|
+ * we should just do "return pfn_to_page(pfn)", but
|
|
|
+ * in the meantime we check that we get a valid pfn,
|
|
|
+ * and that the resulting page looks ok.
|
|
|
+ *
|
|
|
+ * Remove this test eventually!
|
|
|
+ */
|
|
|
+ if (unlikely(!pfn_valid(pfn))) {
|
|
|
+ print_bad_pte(vma, pte, addr);
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * NOTE! We still have PageReserved() pages in the page
|
|
|
+ * tables.
|
|
|
+ *
|
|
|
+ * The PAGE_ZERO() pages and various VDSO mappings can
|
|
|
+ * cause them to exist.
|
|
|
+ */
|
|
|
+ return pfn_to_page(pfn);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -379,7 +416,6 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
|
|
|
unsigned long vm_flags = vma->vm_flags;
|
|
|
pte_t pte = *src_pte;
|
|
|
struct page *page;
|
|
|
- unsigned long pfn;
|
|
|
|
|
|
/* pte contains position in swap or file, so copy. */
|
|
|
if (unlikely(!pte_present(pte))) {
|
|
@@ -397,22 +433,6 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
|
|
|
goto out_set_pte;
|
|
|
}
|
|
|
|
|
|
- pfn = pte_pfn(pte);
|
|
|
- page = pfn_valid(pfn)? pfn_to_page(pfn): NULL;
|
|
|
-
|
|
|
- if (unlikely(vm_flags & VM_UNPAGED))
|
|
|
- if (!page_is_anon(page, vma, addr))
|
|
|
- goto out_set_pte;
|
|
|
-
|
|
|
- /*
|
|
|
- * If the pte points outside of valid memory but
|
|
|
- * the region is not VM_UNPAGED, we have a problem.
|
|
|
- */
|
|
|
- if (unlikely(!page)) {
|
|
|
- print_bad_pte(vma, pte, addr);
|
|
|
- goto out_set_pte; /* try to do something sane */
|
|
|
- }
|
|
|
-
|
|
|
/*
|
|
|
* If it's a COW mapping, write protect it both
|
|
|
* in the parent and the child
|
|
@@ -429,9 +449,13 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
|
|
|
if (vm_flags & VM_SHARED)
|
|
|
pte = pte_mkclean(pte);
|
|
|
pte = pte_mkold(pte);
|
|
|
- get_page(page);
|
|
|
- page_dup_rmap(page);
|
|
|
- rss[!!PageAnon(page)]++;
|
|
|
+
|
|
|
+ page = vm_normal_page(vma, addr, pte);
|
|
|
+ if (page) {
|
|
|
+ get_page(page);
|
|
|
+ page_dup_rmap(page);
|
|
|
+ rss[!!PageAnon(page)]++;
|
|
|
+ }
|
|
|
|
|
|
out_set_pte:
|
|
|
set_pte_at(dst_mm, addr, dst_pte, pte);
|
|
@@ -543,7 +567,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
|
|
|
* readonly mappings. The tradeoff is that copy_page_range is more
|
|
|
* efficient than faulting.
|
|
|
*/
|
|
|
- if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_UNPAGED))) {
|
|
|
+ if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP))) {
|
|
|
if (!vma->anon_vma)
|
|
|
return 0;
|
|
|
}
|
|
@@ -584,19 +608,10 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
|
|
|
}
|
|
|
if (pte_present(ptent)) {
|
|
|
struct page *page;
|
|
|
- unsigned long pfn;
|
|
|
|
|
|
(*zap_work) -= PAGE_SIZE;
|
|
|
|
|
|
- pfn = pte_pfn(ptent);
|
|
|
- page = pfn_valid(pfn)? pfn_to_page(pfn): NULL;
|
|
|
-
|
|
|
- if (unlikely(vma->vm_flags & VM_UNPAGED)) {
|
|
|
- if (!page_is_anon(page, vma, addr))
|
|
|
- page = NULL;
|
|
|
- } else if (unlikely(!page))
|
|
|
- print_bad_pte(vma, ptent, addr);
|
|
|
-
|
|
|
+ page = vm_normal_page(vma, addr, ptent);
|
|
|
if (unlikely(details) && page) {
|
|
|
/*
|
|
|
* unmap_shared_mapping_pages() wants to
|
|
@@ -852,7 +867,7 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
|
|
|
/*
|
|
|
* Do a quick page-table lookup for a single page.
|
|
|
*/
|
|
|
-struct page *follow_page(struct mm_struct *mm, unsigned long address,
|
|
|
+struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
|
|
|
unsigned int flags)
|
|
|
{
|
|
|
pgd_t *pgd;
|
|
@@ -860,8 +875,8 @@ struct page *follow_page(struct mm_struct *mm, unsigned long address,
|
|
|
pmd_t *pmd;
|
|
|
pte_t *ptep, pte;
|
|
|
spinlock_t *ptl;
|
|
|
- unsigned long pfn;
|
|
|
struct page *page;
|
|
|
+ struct mm_struct *mm = vma->vm_mm;
|
|
|
|
|
|
page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
|
|
|
if (!IS_ERR(page)) {
|
|
@@ -897,11 +912,10 @@ struct page *follow_page(struct mm_struct *mm, unsigned long address,
|
|
|
goto unlock;
|
|
|
if ((flags & FOLL_WRITE) && !pte_write(pte))
|
|
|
goto unlock;
|
|
|
- pfn = pte_pfn(pte);
|
|
|
- if (!pfn_valid(pfn))
|
|
|
+ page = vm_normal_page(vma, address, pte);
|
|
|
+ if (unlikely(!page))
|
|
|
goto unlock;
|
|
|
|
|
|
- page = pfn_to_page(pfn);
|
|
|
if (flags & FOLL_GET)
|
|
|
get_page(page);
|
|
|
if (flags & FOLL_TOUCH) {
|
|
@@ -974,8 +988,10 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
|
|
|
return i ? : -EFAULT;
|
|
|
}
|
|
|
if (pages) {
|
|
|
- pages[i] = pte_page(*pte);
|
|
|
- get_page(pages[i]);
|
|
|
+ struct page *page = vm_normal_page(vma, start, *pte);
|
|
|
+ pages[i] = page;
|
|
|
+ if (page)
|
|
|
+ get_page(page);
|
|
|
}
|
|
|
pte_unmap(pte);
|
|
|
if (vmas)
|
|
@@ -1010,7 +1026,7 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
|
|
|
foll_flags |= FOLL_WRITE;
|
|
|
|
|
|
cond_resched();
|
|
|
- while (!(page = follow_page(mm, start, foll_flags))) {
|
|
|
+ while (!(page = follow_page(vma, start, foll_flags))) {
|
|
|
int ret;
|
|
|
ret = __handle_mm_fault(mm, vma, start,
|
|
|
foll_flags & FOLL_WRITE);
|
|
@@ -1214,11 +1230,12 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
|
|
|
* in 2.6 the LRU scan won't even find its pages, so this
|
|
|
* flag means no more than count its pages in reserved_vm,
|
|
|
* and omit it from core dump, even when VM_IO turned off.
|
|
|
- * VM_UNPAGED tells the core MM not to "manage" these pages
|
|
|
- * (e.g. refcount, mapcount, try to swap them out): in
|
|
|
- * particular, zap_pte_range does not try to free them.
|
|
|
+ * VM_PFNMAP tells the core MM that the base pages are just
|
|
|
+ * raw PFN mappings, and do not have a "struct page" associated
|
|
|
+ * with them.
|
|
|
*/
|
|
|
- vma->vm_flags |= VM_IO | VM_RESERVED | VM_UNPAGED;
|
|
|
+ vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
|
|
|
+ vma->vm_pgoff = pfn;
|
|
|
|
|
|
BUG_ON(addr >= end);
|
|
|
pfn -= addr >> PAGE_SHIFT;
|
|
@@ -1273,6 +1290,26 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
|
|
|
return pte;
|
|
|
}
|
|
|
|
|
|
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * If the source page was a PFN mapping, we don't have
|
|
|
+ * a "struct page" for it. We do a best-effort copy by
|
|
|
+ * just copying from the original user address. If that
|
|
|
+ * fails, we just zero-fill it. Live with it.
|
|
|
+ */
|
|
|
+ if (unlikely(!src)) {
|
|
|
+ void *kaddr = kmap_atomic(dst, KM_USER0);
|
|
|
+ unsigned long left = __copy_from_user_inatomic(kaddr, (void __user *)va, PAGE_SIZE);
|
|
|
+ if (left)
|
|
|
+ memset(kaddr, 0, PAGE_SIZE);
|
|
|
+ kunmap_atomic(kaddr, KM_USER0);
|
|
|
+ return;
|
|
|
+
|
|
|
+ }
|
|
|
+ copy_user_highpage(dst, src, va);
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* This routine handles present pages, when users try to write
|
|
|
* to a shared page. It is done by copying the page to a new address
|
|
@@ -1296,28 +1333,13 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
spinlock_t *ptl, pte_t orig_pte)
|
|
|
{
|
|
|
struct page *old_page, *src_page, *new_page;
|
|
|
- unsigned long pfn = pte_pfn(orig_pte);
|
|
|
pte_t entry;
|
|
|
int ret = VM_FAULT_MINOR;
|
|
|
|
|
|
- if (unlikely(!pfn_valid(pfn))) {
|
|
|
- /*
|
|
|
- * Page table corrupted: show pte and kill process.
|
|
|
- * Or it's an attempt to COW an out-of-map VM_UNPAGED
|
|
|
- * entry, which copy_user_highpage does not support.
|
|
|
- */
|
|
|
- print_bad_pte(vma, orig_pte, address);
|
|
|
- ret = VM_FAULT_OOM;
|
|
|
- goto unlock;
|
|
|
- }
|
|
|
- old_page = pfn_to_page(pfn);
|
|
|
+ old_page = vm_normal_page(vma, address, orig_pte);
|
|
|
src_page = old_page;
|
|
|
-
|
|
|
- if (unlikely(vma->vm_flags & VM_UNPAGED))
|
|
|
- if (!page_is_anon(old_page, vma, address)) {
|
|
|
- old_page = NULL;
|
|
|
- goto gotten;
|
|
|
- }
|
|
|
+ if (!old_page)
|
|
|
+ goto gotten;
|
|
|
|
|
|
if (PageAnon(old_page) && !TestSetPageLocked(old_page)) {
|
|
|
int reuse = can_share_swap_page(old_page);
|
|
@@ -1351,7 +1373,7 @@ gotten:
|
|
|
new_page = alloc_page_vma(GFP_HIGHUSER, vma, address);
|
|
|
if (!new_page)
|
|
|
goto oom;
|
|
|
- copy_user_highpage(new_page, src_page, address);
|
|
|
+ cow_user_page(new_page, src_page, address);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1812,16 +1834,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
spinlock_t *ptl;
|
|
|
pte_t entry;
|
|
|
|
|
|
- /*
|
|
|
- * A VM_UNPAGED vma will normally be filled with present ptes
|
|
|
- * by remap_pfn_range, and never arrive here; but it might have
|
|
|
- * holes, or if !VM_DONTEXPAND, mremap might have expanded it.
|
|
|
- * It's weird enough handling anon pages in unpaged vmas, we do
|
|
|
- * not want to worry about ZERO_PAGEs too (it may or may not
|
|
|
- * matter if their counts wrap): just give them anon pages.
|
|
|
- */
|
|
|
-
|
|
|
- if (write_access || (vma->vm_flags & VM_UNPAGED)) {
|
|
|
+ if (write_access) {
|
|
|
/* Allocate our own private page. */
|
|
|
pte_unmap(page_table);
|
|
|
|
|
@@ -1896,8 +1909,6 @@ static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
int anon = 0;
|
|
|
|
|
|
pte_unmap(page_table);
|
|
|
- BUG_ON(vma->vm_flags & VM_UNPAGED);
|
|
|
-
|
|
|
if (vma->vm_file) {
|
|
|
mapping = vma->vm_file->f_mapping;
|
|
|
sequence = mapping->truncate_count;
|
|
@@ -1930,7 +1941,7 @@ retry:
|
|
|
page = alloc_page_vma(GFP_HIGHUSER, vma, address);
|
|
|
if (!page)
|
|
|
goto oom;
|
|
|
- copy_user_highpage(page, new_page, address);
|
|
|
+ cow_user_page(page, new_page, address);
|
|
|
page_cache_release(new_page);
|
|
|
new_page = page;
|
|
|
anon = 1;
|