|
@@ -506,6 +506,7 @@ again:
|
|
|
src_pte = pte_offset_map_nested(src_pmd, addr);
|
|
|
src_ptl = pte_lockptr(src_mm, src_pmd);
|
|
|
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
|
|
|
+ arch_enter_lazy_mmu_mode();
|
|
|
|
|
|
do {
|
|
|
/*
|
|
@@ -527,6 +528,7 @@ again:
|
|
|
progress += 8;
|
|
|
} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
|
|
|
|
|
|
+ arch_leave_lazy_mmu_mode();
|
|
|
spin_unlock(src_ptl);
|
|
|
pte_unmap_nested(src_pte - 1);
|
|
|
add_mm_rss(dst_mm, rss[0], rss[1]);
|
|
@@ -628,6 +630,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
|
|
|
int anon_rss = 0;
|
|
|
|
|
|
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
|
|
|
+ arch_enter_lazy_mmu_mode();
|
|
|
do {
|
|
|
pte_t ptent = *pte;
|
|
|
if (pte_none(ptent)) {
|
|
@@ -694,6 +697,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
|
|
|
} while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
|
|
|
|
|
|
add_mm_rss(mm, file_rss, anon_rss);
|
|
|
+ arch_leave_lazy_mmu_mode();
|
|
|
pte_unmap_unlock(pte - 1, ptl);
|
|
|
|
|
|
return addr;
|
|
@@ -1109,6 +1113,7 @@ static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
|
|
|
pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
|
|
|
if (!pte)
|
|
|
return -ENOMEM;
|
|
|
+ arch_enter_lazy_mmu_mode();
|
|
|
do {
|
|
|
struct page *page = ZERO_PAGE(addr);
|
|
|
pte_t zero_pte = pte_wrprotect(mk_pte(page, prot));
|
|
@@ -1118,6 +1123,7 @@ static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
|
|
|
BUG_ON(!pte_none(*pte));
|
|
|
set_pte_at(mm, addr, pte, zero_pte);
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
+ arch_leave_lazy_mmu_mode();
|
|
|
pte_unmap_unlock(pte - 1, ptl);
|
|
|
return 0;
|
|
|
}
|
|
@@ -1275,11 +1281,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
|
|
|
pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
|
|
|
if (!pte)
|
|
|
return -ENOMEM;
|
|
|
+ arch_enter_lazy_mmu_mode();
|
|
|
do {
|
|
|
BUG_ON(!pte_none(*pte));
|
|
|
set_pte_at(mm, addr, pte, pfn_pte(pfn, prot));
|
|
|
pfn++;
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
+ arch_leave_lazy_mmu_mode();
|
|
|
pte_unmap_unlock(pte - 1, ptl);
|
|
|
return 0;
|
|
|
}
|