|
|
@@ -27,85 +27,161 @@
|
|
|
#include <linux/mman.h>
|
|
|
#include <asm/tlb.h>
|
|
|
#include <asm/tlbflush.h>
|
|
|
+#include <asm/setup.h>
|
|
|
+
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+
|
|
|
+/*
|
|
|
+ * Provide an additional huge page size (in addition to the regular default
|
|
|
+ * huge page size) if no "hugepagesz" arguments are specified.
|
|
|
+ * Note that it must be smaller than the default huge page size so
|
|
|
+ * that it's possible to allocate them on demand from the buddy allocator.
|
|
|
+ * You can change this to 64K (on a 16K build), 256K, 1M, or 4M,
|
|
|
+ * or not define it at all.
|
|
|
+ */
|
|
|
+#define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)
|
|
|
+
|
|
|
+/* "Extra" page-size multipliers, one per level of the page table. */
|
|
|
+int huge_shift[HUGE_SHIFT_ENTRIES] = {
|
|
|
+#ifdef ADDITIONAL_HUGE_SIZE
|
|
|
+#define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
|
|
|
+ [HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
|
|
|
+#endif
|
|
|
+};
|
|
|
+
|
|
|
+/*
|
|
|
+ * This routine is a hybrid of pte_alloc_map() and pte_alloc_kernel().
|
|
|
+ * It assumes that L2 PTEs are never in HIGHMEM (we don't support that).
|
|
|
+ * It locks the user pagetable, and bumps up the mm->nr_ptes field,
|
|
|
+ * but otherwise allocate the page table using the kernel versions.
|
|
|
+ */
|
|
|
+static pte_t *pte_alloc_hugetlb(struct mm_struct *mm, pmd_t *pmd,
|
|
|
+ unsigned long address)
|
|
|
+{
|
|
|
+ pte_t *new;
|
|
|
+
|
|
|
+ if (pmd_none(*pmd)) {
|
|
|
+ new = pte_alloc_one_kernel(mm, address);
|
|
|
+ if (!new)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ smp_wmb(); /* See comment in __pte_alloc */
|
|
|
+
|
|
|
+ spin_lock(&mm->page_table_lock);
|
|
|
+ if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
|
|
|
+ mm->nr_ptes++;
|
|
|
+ pmd_populate_kernel(mm, pmd, new);
|
|
|
+ new = NULL;
|
|
|
+ } else
|
|
|
+ VM_BUG_ON(pmd_trans_splitting(*pmd));
|
|
|
+ spin_unlock(&mm->page_table_lock);
|
|
|
+ if (new)
|
|
|
+ pte_free_kernel(mm, new);
|
|
|
+ }
|
|
|
+
|
|
|
+ return pte_offset_kernel(pmd, address);
|
|
|
+}
|
|
|
+#endif
|
|
|
|
|
|
pte_t *huge_pte_alloc(struct mm_struct *mm,
|
|
|
unsigned long addr, unsigned long sz)
|
|
|
{
|
|
|
pgd_t *pgd;
|
|
|
pud_t *pud;
|
|
|
- pte_t *pte = NULL;
|
|
|
|
|
|
- /* We do not yet support multiple huge page sizes. */
|
|
|
- BUG_ON(sz != PMD_SIZE);
|
|
|
+ addr &= -sz; /* Mask off any low bits in the address. */
|
|
|
|
|
|
pgd = pgd_offset(mm, addr);
|
|
|
pud = pud_alloc(mm, pgd, addr);
|
|
|
- if (pud)
|
|
|
- pte = (pte_t *) pmd_alloc(mm, pud, addr);
|
|
|
- BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
|
|
|
|
|
|
- return pte;
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+ if (sz >= PGDIR_SIZE) {
|
|
|
+ BUG_ON(sz != PGDIR_SIZE &&
|
|
|
+ sz != PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
|
|
|
+ return (pte_t *)pud;
|
|
|
+ } else {
|
|
|
+ pmd_t *pmd = pmd_alloc(mm, pud, addr);
|
|
|
+ if (sz >= PMD_SIZE) {
|
|
|
+ BUG_ON(sz != PMD_SIZE &&
|
|
|
+ sz != (PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
|
|
|
+ return (pte_t *)pmd;
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
|
|
|
+ panic("Unexpected page size %#lx\n", sz);
|
|
|
+ return pte_alloc_hugetlb(mm, pmd, addr);
|
|
|
+ }
|
|
|
+ }
|
|
|
+#else
|
|
|
+ BUG_ON(sz != PMD_SIZE);
|
|
|
+ return (pte_t *) pmd_alloc(mm, pud, addr);
|
|
|
+#endif
|
|
|
}
|
|
|
|
|
|
-pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
|
|
|
+static pte_t *get_pte(pte_t *base, int index, int level)
|
|
|
{
|
|
|
- pgd_t *pgd;
|
|
|
- pud_t *pud;
|
|
|
- pmd_t *pmd = NULL;
|
|
|
-
|
|
|
- pgd = pgd_offset(mm, addr);
|
|
|
- if (pgd_present(*pgd)) {
|
|
|
- pud = pud_offset(pgd, addr);
|
|
|
- if (pud_present(*pud))
|
|
|
- pmd = pmd_offset(pud, addr);
|
|
|
+ pte_t *ptep = base + index;
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+ if (!pte_present(*ptep) && huge_shift[level] != 0) {
|
|
|
+ unsigned long mask = -1UL << huge_shift[level];
|
|
|
+ pte_t *super_ptep = base + (index & mask);
|
|
|
+ pte_t pte = *super_ptep;
|
|
|
+ if (pte_present(pte) && pte_super(pte))
|
|
|
+ ptep = super_ptep;
|
|
|
}
|
|
|
- return (pte_t *) pmd;
|
|
|
+#endif
|
|
|
+ return ptep;
|
|
|
}
|
|
|
|
|
|
-#ifdef HUGETLB_TEST
|
|
|
-struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
|
|
|
- int write)
|
|
|
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
|
|
|
{
|
|
|
- unsigned long start = address;
|
|
|
- int length = 1;
|
|
|
- int nr;
|
|
|
- struct page *page;
|
|
|
- struct vm_area_struct *vma;
|
|
|
-
|
|
|
- vma = find_vma(mm, addr);
|
|
|
- if (!vma || !is_vm_hugetlb_page(vma))
|
|
|
- return ERR_PTR(-EINVAL);
|
|
|
-
|
|
|
- pte = huge_pte_offset(mm, address);
|
|
|
+ pgd_t *pgd;
|
|
|
+ pud_t *pud;
|
|
|
+ pmd_t *pmd;
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+ pte_t *pte;
|
|
|
+#endif
|
|
|
|
|
|
- /* hugetlb should be locked, and hence, prefaulted */
|
|
|
- WARN_ON(!pte || pte_none(*pte));
|
|
|
+ /* Get the top-level page table entry. */
|
|
|
+ pgd = (pgd_t *)get_pte((pte_t *)mm->pgd, pgd_index(addr), 0);
|
|
|
+ if (!pgd_present(*pgd))
|
|
|
+ return NULL;
|
|
|
|
|
|
- page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
|
|
|
+ /* We don't have four levels. */
|
|
|
+ pud = pud_offset(pgd, addr);
|
|
|
+#ifndef __PAGETABLE_PUD_FOLDED
|
|
|
+# error support fourth page table level
|
|
|
+#endif
|
|
|
|
|
|
- WARN_ON(!PageHead(page));
|
|
|
+ /* Check for an L0 huge PTE, if we have three levels. */
|
|
|
+#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
+ if (pud_huge(*pud))
|
|
|
+ return (pte_t *)pud;
|
|
|
|
|
|
- return page;
|
|
|
-}
|
|
|
-
|
|
|
-int pmd_huge(pmd_t pmd)
|
|
|
-{
|
|
|
- return 0;
|
|
|
-}
|
|
|
+ pmd = (pmd_t *)get_pte((pte_t *)pud_page_vaddr(*pud),
|
|
|
+ pmd_index(addr), 1);
|
|
|
+ if (!pmd_present(*pmd))
|
|
|
+ return NULL;
|
|
|
+#else
|
|
|
+ pmd = pmd_offset(pud, addr);
|
|
|
+#endif
|
|
|
|
|
|
-int pud_huge(pud_t pud)
|
|
|
-{
|
|
|
- return 0;
|
|
|
-}
|
|
|
+ /* Check for an L1 huge PTE. */
|
|
|
+ if (pmd_huge(*pmd))
|
|
|
+ return (pte_t *)pmd;
|
|
|
+
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+ /* Check for an L2 huge PTE. */
|
|
|
+ pte = get_pte((pte_t *)pmd_page_vaddr(*pmd), pte_index(addr), 2);
|
|
|
+ if (!pte_present(*pte))
|
|
|
+ return NULL;
|
|
|
+ if (pte_super(*pte))
|
|
|
+ return pte;
|
|
|
+#endif
|
|
|
|
|
|
-struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
|
|
- pmd_t *pmd, int write)
|
|
|
-{
|
|
|
return NULL;
|
|
|
}
|
|
|
|
|
|
-#else
|
|
|
-
|
|
|
struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
|
|
|
int write)
|
|
|
{
|
|
|
@@ -149,8 +225,6 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-#endif
|
|
|
-
|
|
|
#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
|
|
|
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
|
|
|
unsigned long addr, unsigned long len,
|
|
|
@@ -322,21 +396,102 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
|
|
return hugetlb_get_unmapped_area_topdown(file, addr, len,
|
|
|
pgoff, flags);
|
|
|
}
|
|
|
+#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
|
|
|
|
|
|
-static __init int setup_hugepagesz(char *opt)
|
|
|
+#ifdef CONFIG_HUGETLB_SUPER_PAGES
|
|
|
+static __init int __setup_hugepagesz(unsigned long ps)
|
|
|
{
|
|
|
- unsigned long ps = memparse(opt, &opt);
|
|
|
- if (ps == PMD_SIZE) {
|
|
|
- hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
|
|
|
- } else if (ps == PUD_SIZE) {
|
|
|
- hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
|
|
|
+ int log_ps = __builtin_ctzl(ps);
|
|
|
+ int level, base_shift;
|
|
|
+
|
|
|
+ if ((1UL << log_ps) != ps || (log_ps & 1) != 0) {
|
|
|
+ pr_warn("Not enabling %ld byte huge pages;"
|
|
|
+ " must be a power of four.\n", ps);
|
|
|
+ return -EINVAL;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (ps > 64*1024*1024*1024UL) {
|
|
|
+ pr_warn("Not enabling %ld MB huge pages;"
|
|
|
+ " largest legal value is 64 GB .\n", ps >> 20);
|
|
|
+ return -EINVAL;
|
|
|
+ } else if (ps >= PUD_SIZE) {
|
|
|
+ static long hv_jpage_size;
|
|
|
+ if (hv_jpage_size == 0)
|
|
|
+ hv_jpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO);
|
|
|
+ if (hv_jpage_size != PUD_SIZE) {
|
|
|
+ pr_warn("Not enabling >= %ld MB huge pages:"
|
|
|
+ " hypervisor reports size %ld\n",
|
|
|
+ PUD_SIZE >> 20, hv_jpage_size);
|
|
|
+ return -EINVAL;
|
|
|
+ }
|
|
|
+ level = 0;
|
|
|
+ base_shift = PUD_SHIFT;
|
|
|
+ } else if (ps >= PMD_SIZE) {
|
|
|
+ level = 1;
|
|
|
+ base_shift = PMD_SHIFT;
|
|
|
+ } else if (ps > PAGE_SIZE) {
|
|
|
+ level = 2;
|
|
|
+ base_shift = PAGE_SHIFT;
|
|
|
} else {
|
|
|
- pr_err("hugepagesz: Unsupported page size %lu M\n",
|
|
|
- ps >> 20);
|
|
|
- return 0;
|
|
|
+ pr_err("hugepagesz: huge page size %ld too small\n", ps);
|
|
|
+ return -EINVAL;
|
|
|
}
|
|
|
- return 1;
|
|
|
+
|
|
|
+ if (log_ps != base_shift) {
|
|
|
+ int shift_val = log_ps - base_shift;
|
|
|
+ if (huge_shift[level] != 0) {
|
|
|
+ int old_shift = base_shift + huge_shift[level];
|
|
|
+ pr_warn("Not enabling %ld MB huge pages;"
|
|
|
+ " already have size %ld MB.\n",
|
|
|
+ ps >> 20, (1UL << old_shift) >> 20);
|
|
|
+ return -EINVAL;
|
|
|
+ }
|
|
|
+ if (hv_set_pte_super_shift(level, shift_val) != 0) {
|
|
|
+ pr_warn("Not enabling %ld MB huge pages;"
|
|
|
+ " no hypervisor support.\n", ps >> 20);
|
|
|
+ return -EINVAL;
|
|
|
+ }
|
|
|
+ printk(KERN_DEBUG "Enabled %ld MB huge pages\n", ps >> 20);
|
|
|
+ huge_shift[level] = shift_val;
|
|
|
+ }
|
|
|
+
|
|
|
+ hugetlb_add_hstate(log_ps - PAGE_SHIFT);
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static bool saw_hugepagesz;
|
|
|
+
|
|
|
+static __init int setup_hugepagesz(char *opt)
|
|
|
+{
|
|
|
+ if (!saw_hugepagesz) {
|
|
|
+ saw_hugepagesz = true;
|
|
|
+ memset(huge_shift, 0, sizeof(huge_shift));
|
|
|
+ }
|
|
|
+ return __setup_hugepagesz(memparse(opt, NULL));
|
|
|
}
|
|
|
__setup("hugepagesz=", setup_hugepagesz);
|
|
|
|
|
|
-#endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/
|
|
|
+#ifdef ADDITIONAL_HUGE_SIZE
|
|
|
+/*
|
|
|
+ * Provide an additional huge page size if no "hugepagesz" args are given.
|
|
|
+ * In that case, all the cores have properly set up their hv super_shift
|
|
|
+ * already, but we need to notify the hugetlb code to enable the
|
|
|
+ * new huge page size from the Linux point of view.
|
|
|
+ */
|
|
|
+static __init int add_default_hugepagesz(void)
|
|
|
+{
|
|
|
+ if (!saw_hugepagesz) {
|
|
|
+ BUILD_BUG_ON(ADDITIONAL_HUGE_SIZE >= PMD_SIZE ||
|
|
|
+ ADDITIONAL_HUGE_SIZE <= PAGE_SIZE);
|
|
|
+ BUILD_BUG_ON((PAGE_SIZE << ADDITIONAL_HUGE_SHIFT) !=
|
|
|
+ ADDITIONAL_HUGE_SIZE);
|
|
|
+ BUILD_BUG_ON(ADDITIONAL_HUGE_SHIFT & 1);
|
|
|
+ hugetlb_add_hstate(ADDITIONAL_HUGE_SHIFT);
|
|
|
+ }
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+arch_initcall(add_default_hugepagesz);
|
|
|
+#endif
|
|
|
+
|
|
|
+#endif /* CONFIG_HUGETLB_SUPER_PAGES */
|
Chris Metcalf