|
@@ -26,8 +26,10 @@
|
|
|
#include <linux/smp.h>
|
|
|
#include <linux/string.h>
|
|
|
#include <linux/init.h>
|
|
|
+#include <linux/cache.h>
|
|
|
|
|
|
-#include <asm/mmu_context.h>
|
|
|
+#include <asm/cacheflush.h>
|
|
|
+#include <asm/pgtable.h>
|
|
|
#include <asm/war.h>
|
|
|
#include <asm/uasm.h>
|
|
|
|
|
@@ -63,6 +65,52 @@ static inline int __maybe_unused r10000_llsc_war(void)
|
|
|
return R10000_LLSC_WAR;
|
|
|
}
|
|
|
|
|
|
+static int use_bbit_insns(void)
|
|
|
+{
|
|
|
+ switch (current_cpu_type()) {
|
|
|
+ case CPU_CAVIUM_OCTEON:
|
|
|
+ case CPU_CAVIUM_OCTEON_PLUS:
|
|
|
+ case CPU_CAVIUM_OCTEON2:
|
|
|
+ return 1;
|
|
|
+ default:
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static int use_lwx_insns(void)
|
|
|
+{
|
|
|
+ switch (current_cpu_type()) {
|
|
|
+ case CPU_CAVIUM_OCTEON2:
|
|
|
+ return 1;
|
|
|
+ default:
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+}
|
|
|
+#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
|
|
|
+ CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
|
|
|
+static bool scratchpad_available(void)
|
|
|
+{
|
|
|
+ return true;
|
|
|
+}
|
|
|
+static int scratchpad_offset(int i)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * CVMSEG starts at address -32768 and extends for
|
|
|
+ * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
|
|
|
+ */
|
|
|
+ i += 1; /* Kernel use starts at the top and works down. */
|
|
|
+ return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
|
|
|
+}
|
|
|
+#else
|
|
|
+static bool scratchpad_available(void)
|
|
|
+{
|
|
|
+ return false;
|
|
|
+}
|
|
|
+static int scratchpad_offset(int i)
|
|
|
+{
|
|
|
+ BUG();
|
|
|
+}
|
|
|
+#endif
|
|
|
/*
|
|
|
* Found by experiment: At least some revisions of the 4kc throw under
|
|
|
* some circumstances a machine check exception, triggered by invalid
|
|
@@ -173,11 +221,41 @@ static struct uasm_reloc relocs[128] __cpuinitdata;
|
|
|
static int check_for_high_segbits __cpuinitdata;
|
|
|
#endif
|
|
|
|
|
|
+static int check_for_high_segbits __cpuinitdata;
|
|
|
+
|
|
|
+static unsigned int kscratch_used_mask __cpuinitdata;
|
|
|
+
|
|
|
+static int __cpuinit allocate_kscratch(void)
|
|
|
+{
|
|
|
+ int r;
|
|
|
+ unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
|
|
|
+
|
|
|
+ r = ffs(a);
|
|
|
+
|
|
|
+ if (r == 0)
|
|
|
+ return -1;
|
|
|
+
|
|
|
+ r--; /* make it zero based */
|
|
|
+
|
|
|
+ kscratch_used_mask |= (1 << r);
|
|
|
+
|
|
|
+ return r;
|
|
|
+}
|
|
|
+
|
|
|
+static int scratch_reg __cpuinitdata;
|
|
|
+static int pgd_reg __cpuinitdata;
|
|
|
+enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
|
|
|
+
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
+
|
|
|
/*
|
|
|
* CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
|
|
|
* we cannot do r3000 under these circumstances.
|
|
|
+ *
|
|
|
+ * Declare pgd_current here instead of including mmu_context.h to avoid type
|
|
|
+ * conflicts for tlbmiss_handler_setup_pgd
|
|
|
*/
|
|
|
+extern unsigned long pgd_current[];
|
|
|
|
|
|
/*
|
|
|
* The R3000 TLB handler is simple.
|
|
@@ -440,21 +518,43 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
|
|
|
static __cpuinit void build_restore_pagemask(u32 **p,
|
|
|
struct uasm_reloc **r,
|
|
|
unsigned int tmp,
|
|
|
- enum label_id lid)
|
|
|
+ enum label_id lid,
|
|
|
+ int restore_scratch)
|
|
|
{
|
|
|
- /* Reset default page size */
|
|
|
- if (PM_DEFAULT_MASK >> 16) {
|
|
|
- uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
|
- uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
|
|
|
- uasm_il_b(p, r, lid);
|
|
|
- uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
- } else if (PM_DEFAULT_MASK) {
|
|
|
- uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
|
|
|
- uasm_il_b(p, r, lid);
|
|
|
- uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
+ if (restore_scratch) {
|
|
|
+ /* Reset default page size */
|
|
|
+ if (PM_DEFAULT_MASK >> 16) {
|
|
|
+ uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
|
+ uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
|
|
|
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ } else if (PM_DEFAULT_MASK) {
|
|
|
+ uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
|
|
|
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ } else {
|
|
|
+ uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ }
|
|
|
+ if (scratch_reg > 0)
|
|
|
+ UASM_i_MFC0(p, 1, 31, scratch_reg);
|
|
|
+ else
|
|
|
+ UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
|
|
} else {
|
|
|
- uasm_il_b(p, r, lid);
|
|
|
- uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
|
+ /* Reset default page size */
|
|
|
+ if (PM_DEFAULT_MASK >> 16) {
|
|
|
+ uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
|
|
|
+ uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
+ } else if (PM_DEFAULT_MASK) {
|
|
|
+ uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
|
|
|
+ } else {
|
|
|
+ uasm_il_b(p, r, lid);
|
|
|
+ uasm_i_mtc0(p, 0, C0_PAGEMASK);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -462,7 +562,8 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p,
|
|
|
struct uasm_label **l,
|
|
|
struct uasm_reloc **r,
|
|
|
unsigned int tmp,
|
|
|
- enum tlb_write_entry wmode)
|
|
|
+ enum tlb_write_entry wmode,
|
|
|
+ int restore_scratch)
|
|
|
{
|
|
|
/* Set huge page tlb entry size */
|
|
|
uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
|
|
@@ -471,7 +572,7 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p,
|
|
|
|
|
|
build_tlb_write_entry(p, l, r, wmode);
|
|
|
|
|
|
- build_restore_pagemask(p, r, tmp, label_leave);
|
|
|
+ build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -482,8 +583,12 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
|
|
|
unsigned int pmd, int lid)
|
|
|
{
|
|
|
UASM_i_LW(p, tmp, 0, pmd);
|
|
|
- uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
|
|
|
- uasm_il_bnez(p, r, tmp, lid);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
|
|
|
+ uasm_il_bnez(p, r, tmp, lid);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
static __cpuinit void build_huge_update_entries(u32 **p,
|
|
@@ -532,7 +637,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p,
|
|
|
UASM_i_SW(p, pte, 0, ptr);
|
|
|
#endif
|
|
|
build_huge_update_entries(p, pte, ptr);
|
|
|
- build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed);
|
|
|
+ build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
|
|
|
}
|
|
|
#endif /* CONFIG_HUGETLB_PAGE */
|
|
|
|
|
@@ -573,13 +678,22 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
|
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
|
|
|
|
|
|
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
- /*
|
|
|
- * &pgd << 11 stored in CONTEXT [23..63].
|
|
|
- */
|
|
|
- UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
- uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
|
|
|
- uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */
|
|
|
- uasm_i_drotr(p, ptr, ptr, 11);
|
|
|
+ if (pgd_reg != -1) {
|
|
|
+ /* pgd is in pgd_reg */
|
|
|
+ UASM_i_MFC0(p, ptr, 31, pgd_reg);
|
|
|
+ } else {
|
|
|
+ /*
|
|
|
+ * &pgd << 11 stored in CONTEXT [23..63].
|
|
|
+ */
|
|
|
+ UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
+
|
|
|
+ /* Clear lower 23 bits of context. */
|
|
|
+ uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
+
|
|
|
+ /* 1 0 1 0 1 << 6 xkphys cached */
|
|
|
+ uasm_i_ori(p, ptr, ptr, 0x540);
|
|
|
+ uasm_i_drotr(p, ptr, ptr, 11);
|
|
|
+ }
|
|
|
#elif defined(CONFIG_SMP)
|
|
|
# ifdef CONFIG_MIPS_MT_SMTC
|
|
|
/*
|
|
@@ -620,7 +734,6 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
-enum vmalloc64_mode {not_refill, refill};
|
|
|
/*
|
|
|
* BVADDR is the faulting address, PTR is scratch.
|
|
|
* PTR will hold the pgd for vmalloc.
|
|
@@ -638,7 +751,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
|
|
|
|
uasm_l_vmalloc(l, *p);
|
|
|
|
|
|
- if (mode == refill && check_for_high_segbits) {
|
|
|
+ if (mode != not_refill && check_for_high_segbits) {
|
|
|
if (single_insn_swpd) {
|
|
|
uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
|
|
|
uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
|
|
@@ -661,7 +774,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
|
uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
|
|
|
}
|
|
|
}
|
|
|
- if (mode == refill && check_for_high_segbits) {
|
|
|
+ if (mode != not_refill && check_for_high_segbits) {
|
|
|
uasm_l_large_segbits_fault(l, *p);
|
|
|
/*
|
|
|
* We get here if we are an xsseg address, or if we are
|
|
@@ -677,7 +790,15 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
|
|
|
*/
|
|
|
UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
|
|
|
uasm_i_jr(p, ptr);
|
|
|
- uasm_i_nop(p);
|
|
|
+
|
|
|
+ if (mode == refill_scratch) {
|
|
|
+ if (scratch_reg > 0)
|
|
|
+ UASM_i_MFC0(p, 1, 31, scratch_reg);
|
|
|
+ else
|
|
|
+ UASM_i_LW(p, 1, scratchpad_offset(0), 0);
|
|
|
+ } else {
|
|
|
+ uasm_i_nop(p);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -834,6 +955,185 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
+struct mips_huge_tlb_info {
|
|
|
+ int huge_pte;
|
|
|
+ int restore_scratch;
|
|
|
+};
|
|
|
+
|
|
|
+static struct mips_huge_tlb_info __cpuinit
|
|
|
+build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
|
|
|
+ struct uasm_reloc **r, unsigned int tmp,
|
|
|
+ unsigned int ptr, int c0_scratch)
|
|
|
+{
|
|
|
+ struct mips_huge_tlb_info rv;
|
|
|
+ unsigned int even, odd;
|
|
|
+ int vmalloc_branch_delay_filled = 0;
|
|
|
+ const int scratch = 1; /* Our extra working register */
|
|
|
+
|
|
|
+ rv.huge_pte = scratch;
|
|
|
+ rv.restore_scratch = 0;
|
|
|
+
|
|
|
+ if (check_for_high_segbits) {
|
|
|
+ UASM_i_MFC0(p, tmp, C0_BADVADDR);
|
|
|
+
|
|
|
+ if (pgd_reg != -1)
|
|
|
+ UASM_i_MFC0(p, ptr, 31, pgd_reg);
|
|
|
+ else
|
|
|
+ UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
+
|
|
|
+ if (c0_scratch >= 0)
|
|
|
+ UASM_i_MTC0(p, scratch, 31, c0_scratch);
|
|
|
+ else
|
|
|
+ UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
|
|
|
+
|
|
|
+ uasm_i_dsrl_safe(p, scratch, tmp,
|
|
|
+ PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
|
|
|
+ uasm_il_bnez(p, r, scratch, label_vmalloc);
|
|
|
+
|
|
|
+ if (pgd_reg == -1) {
|
|
|
+ vmalloc_branch_delay_filled = 1;
|
|
|
+ /* Clear lower 23 bits of context. */
|
|
|
+ uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ if (pgd_reg != -1)
|
|
|
+ UASM_i_MFC0(p, ptr, 31, pgd_reg);
|
|
|
+ else
|
|
|
+ UASM_i_MFC0(p, ptr, C0_CONTEXT);
|
|
|
+
|
|
|
+ UASM_i_MFC0(p, tmp, C0_BADVADDR);
|
|
|
+
|
|
|
+ if (c0_scratch >= 0)
|
|
|
+ UASM_i_MTC0(p, scratch, 31, c0_scratch);
|
|
|
+ else
|
|
|
+ UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
|
|
|
+
|
|
|
+ if (pgd_reg == -1)
|
|
|
+ /* Clear lower 23 bits of context. */
|
|
|
+ uasm_i_dins(p, ptr, 0, 0, 23);
|
|
|
+
|
|
|
+ uasm_il_bltz(p, r, tmp, label_vmalloc);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (pgd_reg == -1) {
|
|
|
+ vmalloc_branch_delay_filled = 1;
|
|
|
+ /* 1 0 1 0 1 << 6 xkphys cached */
|
|
|
+ uasm_i_ori(p, ptr, ptr, 0x540);
|
|
|
+ uasm_i_drotr(p, ptr, ptr, 11);
|
|
|
+ }
|
|
|
+
|
|
|
+#ifdef __PAGETABLE_PMD_FOLDED
|
|
|
+#define LOC_PTEP scratch
|
|
|
+#else
|
|
|
+#define LOC_PTEP ptr
|
|
|
+#endif
|
|
|
+
|
|
|
+ if (!vmalloc_branch_delay_filled)
|
|
|
+ /* get pgd offset in bytes */
|
|
|
+ uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
|
|
|
+
|
|
|
+ uasm_l_vmalloc_done(l, *p);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * tmp ptr
|
|
|
+ * fall-through case = badvaddr *pgd_current
|
|
|
+ * vmalloc case = badvaddr swapper_pg_dir
|
|
|
+ */
|
|
|
+
|
|
|
+ if (vmalloc_branch_delay_filled)
|
|
|
+ /* get pgd offset in bytes */
|
|
|
+ uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
|
|
|
+
|
|
|
+#ifdef __PAGETABLE_PMD_FOLDED
|
|
|
+ GET_CONTEXT(p, tmp); /* get context reg */
|
|
|
+#endif
|
|
|
+ uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
|
|
|
+
|
|
|
+ if (use_lwx_insns()) {
|
|
|
+ UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
|
|
|
+ } else {
|
|
|
+ uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
|
|
|
+ uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
|
|
|
+ }
|
|
|
+
|
|
|
+#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
+ /* get pmd offset in bytes */
|
|
|
+ uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
|
|
|
+ uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
|
|
|
+ GET_CONTEXT(p, tmp); /* get context reg */
|
|
|
+
|
|
|
+ if (use_lwx_insns()) {
|
|
|
+ UASM_i_LWX(p, scratch, scratch, ptr);
|
|
|
+ } else {
|
|
|
+ uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
|
|
|
+ UASM_i_LW(p, scratch, 0, ptr);
|
|
|
+ }
|
|
|
+#endif
|
|
|
+ /* Adjust the context during the load latency. */
|
|
|
+ build_adjust_context(p, tmp);
|
|
|
+
|
|
|
+#ifdef CONFIG_HUGETLB_PAGE
|
|
|
+ uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
|
|
|
+ /*
|
|
|
+ * The in the LWX case we don't want to do the load in the
|
|
|
+ * delay slot. It cannot issue in the same cycle and may be
|
|
|
+ * speculative and unneeded.
|
|
|
+ */
|
|
|
+ if (use_lwx_insns())
|
|
|
+ uasm_i_nop(p);
|
|
|
+#endif /* CONFIG_HUGETLB_PAGE */
|
|
|
+
|
|
|
+
|
|
|
+ /* build_update_entries */
|
|
|
+ if (use_lwx_insns()) {
|
|
|
+ even = ptr;
|
|
|
+ odd = tmp;
|
|
|
+ UASM_i_LWX(p, even, scratch, tmp);
|
|
|
+ UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
|
|
|
+ UASM_i_LWX(p, odd, scratch, tmp);
|
|
|
+ } else {
|
|
|
+ UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
|
|
|
+ even = tmp;
|
|
|
+ odd = ptr;
|
|
|
+ UASM_i_LW(p, even, 0, ptr); /* get even pte */
|
|
|
+ UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
|
|
|
+ }
|
|
|
+ if (kernel_uses_smartmips_rixi) {
|
|
|
+ uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC));
|
|
|
+ uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC));
|
|
|
+ uasm_i_drotr(p, even, even,
|
|
|
+ ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
|
|
|
+ UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
|
|
|
+ uasm_i_drotr(p, odd, odd,
|
|
|
+ ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
|
|
|
+ } else {
|
|
|
+ uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
|
|
|
+ UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
|
|
|
+ uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
|
|
|
+ }
|
|
|
+ UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
|
|
|
+
|
|
|
+ if (c0_scratch >= 0) {
|
|
|
+ UASM_i_MFC0(p, scratch, 31, c0_scratch);
|
|
|
+ build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
+ uasm_l_leave(l, *p);
|
|
|
+ rv.restore_scratch = 1;
|
|
|
+ } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
|
|
|
+ build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
+ uasm_l_leave(l, *p);
|
|
|
+ UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
|
|
|
+ } else {
|
|
|
+ UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
|
|
|
+ build_tlb_write_entry(p, l, r, tlb_random);
|
|
|
+ uasm_l_leave(l, *p);
|
|
|
+ rv.restore_scratch = 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ uasm_i_eret(p); /* return from trap */
|
|
|
+
|
|
|
+ return rv;
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* For a 64-bit kernel, we are using the 64-bit XTLB refill exception
|
|
|
* because EXL == 0. If we wrap, we can also use the 32 instruction
|
|
@@ -849,54 +1149,67 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
|
|
|
struct uasm_reloc *r = relocs;
|
|
|
u32 *f;
|
|
|
unsigned int final_len;
|
|
|
+ struct mips_huge_tlb_info htlb_info;
|
|
|
+ enum vmalloc64_mode vmalloc_mode;
|
|
|
|
|
|
memset(tlb_handler, 0, sizeof(tlb_handler));
|
|
|
memset(labels, 0, sizeof(labels));
|
|
|
memset(relocs, 0, sizeof(relocs));
|
|
|
memset(final_handler, 0, sizeof(final_handler));
|
|
|
|
|
|
- /*
|
|
|
- * create the plain linear handler
|
|
|
- */
|
|
|
- if (bcm1250_m3_war()) {
|
|
|
- unsigned int segbits = 44;
|
|
|
+ if (scratch_reg == 0)
|
|
|
+ scratch_reg = allocate_kscratch();
|
|
|
|
|
|
- uasm_i_dmfc0(&p, K0, C0_BADVADDR);
|
|
|
- uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
|
|
|
- uasm_i_xor(&p, K0, K0, K1);
|
|
|
- uasm_i_dsrl_safe(&p, K1, K0, 62);
|
|
|
- uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
|
|
|
- uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
|
|
|
- uasm_i_or(&p, K0, K0, K1);
|
|
|
- uasm_il_bnez(&p, &r, K0, label_leave);
|
|
|
- /* No need for uasm_i_nop */
|
|
|
- }
|
|
|
+ if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) {
|
|
|
+ htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
|
|
|
+ scratch_reg);
|
|
|
+ vmalloc_mode = refill_scratch;
|
|
|
+ } else {
|
|
|
+ htlb_info.huge_pte = K0;
|
|
|
+ htlb_info.restore_scratch = 0;
|
|
|
+ vmalloc_mode = refill_noscratch;
|
|
|
+ /*
|
|
|
+ * create the plain linear handler
|
|
|
+ */
|
|
|
+ if (bcm1250_m3_war()) {
|
|
|
+ unsigned int segbits = 44;
|
|
|
+
|
|
|
+ uasm_i_dmfc0(&p, K0, C0_BADVADDR);
|
|
|
+ uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
|
|
|
+ uasm_i_xor(&p, K0, K0, K1);
|
|
|
+ uasm_i_dsrl_safe(&p, K1, K0, 62);
|
|
|
+ uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
|
|
|
+ uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
|
|
|
+ uasm_i_or(&p, K0, K0, K1);
|
|
|
+ uasm_il_bnez(&p, &r, K0, label_leave);
|
|
|
+ /* No need for uasm_i_nop */
|
|
|
+ }
|
|
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
- build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
|
|
|
+ build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
|
|
|
#else
|
|
|
- build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
|
|
|
+ build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
- build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
|
|
|
+ build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
|
|
|
#endif
|
|
|
|
|
|
- build_get_ptep(&p, K0, K1);
|
|
|
- build_update_entries(&p, K0, K1);
|
|
|
- build_tlb_write_entry(&p, &l, &r, tlb_random);
|
|
|
- uasm_l_leave(&l, p);
|
|
|
- uasm_i_eret(&p); /* return from trap */
|
|
|
-
|
|
|
+ build_get_ptep(&p, K0, K1);
|
|
|
+ build_update_entries(&p, K0, K1);
|
|
|
+ build_tlb_write_entry(&p, &l, &r, tlb_random);
|
|
|
+ uasm_l_leave(&l, p);
|
|
|
+ uasm_i_eret(&p); /* return from trap */
|
|
|
+ }
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
uasm_l_tlb_huge_update(&l, p);
|
|
|
- UASM_i_LW(&p, K0, 0, K1);
|
|
|
- build_huge_update_entries(&p, K0, K1);
|
|
|
- build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random);
|
|
|
+ build_huge_update_entries(&p, htlb_info.huge_pte, K1);
|
|
|
+ build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
|
|
|
+ htlb_info.restore_scratch);
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
- build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill);
|
|
|
+ build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
|
|
|
#endif
|
|
|
|
|
|
/*
|
|
@@ -1014,6 +1327,55 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
|
|
|
u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
|
|
|
u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
|
|
|
u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
|
|
|
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
+u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
|
|
|
+
|
|
|
+static void __cpuinit build_r4000_setup_pgd(void)
|
|
|
+{
|
|
|
+ const int a0 = 4;
|
|
|
+ const int a1 = 5;
|
|
|
+ u32 *p = tlbmiss_handler_setup_pgd;
|
|
|
+ struct uasm_label *l = labels;
|
|
|
+ struct uasm_reloc *r = relocs;
|
|
|
+
|
|
|
+ memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
|
|
|
+ memset(labels, 0, sizeof(labels));
|
|
|
+ memset(relocs, 0, sizeof(relocs));
|
|
|
+
|
|
|
+ pgd_reg = allocate_kscratch();
|
|
|
+
|
|
|
+ if (pgd_reg == -1) {
|
|
|
+ /* PGD << 11 in c0_Context */
|
|
|
+ /*
|
|
|
+ * If it is a ckseg0 address, convert to a physical
|
|
|
+ * address. Shifting right by 29 and adding 4 will
|
|
|
+ * result in zero for these addresses.
|
|
|
+ *
|
|
|
+ */
|
|
|
+ UASM_i_SRA(&p, a1, a0, 29);
|
|
|
+ UASM_i_ADDIU(&p, a1, a1, 4);
|
|
|
+ uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
|
|
|
+ uasm_i_nop(&p);
|
|
|
+ uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
|
|
|
+ uasm_l_tlbl_goaround1(&l, p);
|
|
|
+ UASM_i_SLL(&p, a0, a0, 11);
|
|
|
+ uasm_i_jr(&p, 31);
|
|
|
+ UASM_i_MTC0(&p, a0, C0_CONTEXT);
|
|
|
+ } else {
|
|
|
+ /* PGD in c0_KScratch */
|
|
|
+ uasm_i_jr(&p, 31);
|
|
|
+ UASM_i_MTC0(&p, a0, 31, pgd_reg);
|
|
|
+ }
|
|
|
+ if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
|
|
|
+ panic("tlbmiss_handler_setup_pgd space exceeded");
|
|
|
+ uasm_resolve_relocs(relocs, labels);
|
|
|
+ pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
|
|
|
+ (unsigned int)(p - tlbmiss_handler_setup_pgd));
|
|
|
+
|
|
|
+ dump_handler(tlbmiss_handler_setup_pgd,
|
|
|
+ ARRAY_SIZE(tlbmiss_handler_setup_pgd));
|
|
|
+}
|
|
|
+#endif
|
|
|
|
|
|
static void __cpuinit
|
|
|
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
|
|
@@ -1100,14 +1462,20 @@ build_pte_present(u32 **p, struct uasm_reloc **r,
|
|
|
unsigned int pte, unsigned int ptr, enum label_id lid)
|
|
|
{
|
|
|
if (kernel_uses_smartmips_rixi) {
|
|
|
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
|
|
|
- uasm_il_beqz(p, r, pte, lid);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
|
|
|
+ uasm_i_nop(p);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
|
|
|
+ uasm_il_beqz(p, r, pte, lid);
|
|
|
+ iPTE_LW(p, pte, ptr);
|
|
|
+ }
|
|
|
} else {
|
|
|
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
|
|
|
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
|
|
|
uasm_il_bnez(p, r, pte, lid);
|
|
|
+ iPTE_LW(p, pte, ptr);
|
|
|
}
|
|
|
- iPTE_LW(p, pte, ptr);
|
|
|
}
|
|
|
|
|
|
/* Make PTE valid, store result in PTR. */
|
|
@@ -1128,10 +1496,17 @@ static void __cpuinit
|
|
|
build_pte_writable(u32 **p, struct uasm_reloc **r,
|
|
|
unsigned int pte, unsigned int ptr, enum label_id lid)
|
|
|
{
|
|
|
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
|
|
|
- uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
|
|
|
- uasm_il_bnez(p, r, pte, lid);
|
|
|
- iPTE_LW(p, pte, ptr);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
|
|
|
+ uasm_i_nop(p);
|
|
|
+ uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
|
|
|
+ uasm_i_nop(p);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
|
|
|
+ uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
|
|
|
+ uasm_il_bnez(p, r, pte, lid);
|
|
|
+ iPTE_LW(p, pte, ptr);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/* Make PTE writable, update software status bits as well, then store
|
|
@@ -1155,12 +1530,19 @@ static void __cpuinit
|
|
|
build_pte_modifiable(u32 **p, struct uasm_reloc **r,
|
|
|
unsigned int pte, unsigned int ptr, enum label_id lid)
|
|
|
{
|
|
|
- uasm_i_andi(p, pte, pte, _PAGE_WRITE);
|
|
|
- uasm_il_beqz(p, r, pte, lid);
|
|
|
- iPTE_LW(p, pte, ptr);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
|
|
|
+ uasm_i_nop(p);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(p, pte, pte, _PAGE_WRITE);
|
|
|
+ uasm_il_beqz(p, r, pte, lid);
|
|
|
+ iPTE_LW(p, pte, ptr);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
+
|
|
|
+
|
|
|
/*
|
|
|
* R3000 style TLB load/store/modify handlers.
|
|
|
*/
|
|
@@ -1402,14 +1784,23 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
|
|
|
* If the page is not _PAGE_VALID, RI or XI could not
|
|
|
* have triggered it. Skip the expensive test..
|
|
|
*/
|
|
|
- uasm_i_andi(&p, K0, K0, _PAGE_VALID);
|
|
|
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
|
|
|
+ label_tlbl_goaround1);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K0, _PAGE_VALID);
|
|
|
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
|
|
|
+ }
|
|
|
uasm_i_nop(&p);
|
|
|
|
|
|
uasm_i_tlbr(&p);
|
|
|
/* Examine entrylo 0 or 1 based on ptr. */
|
|
|
- uasm_i_andi(&p, K0, K1, sizeof(pte_t));
|
|
|
- uasm_i_beqz(&p, K0, 8);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K1, sizeof(pte_t));
|
|
|
+ uasm_i_beqz(&p, K0, 8);
|
|
|
+ }
|
|
|
|
|
|
UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
|
|
|
UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
|
|
@@ -1417,12 +1808,18 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
|
|
|
* If the entryLo (now in K0) is valid (bit 1), RI or
|
|
|
* XI must have triggered it.
|
|
|
*/
|
|
|
- uasm_i_andi(&p, K0, K0, 2);
|
|
|
- uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
|
|
|
-
|
|
|
- uasm_l_tlbl_goaround1(&l, p);
|
|
|
- /* Reload the PTE value */
|
|
|
- iPTE_LW(&p, K0, K1);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl);
|
|
|
+ /* Reload the PTE value */
|
|
|
+ iPTE_LW(&p, K0, K1);
|
|
|
+ uasm_l_tlbl_goaround1(&l, p);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K0, 2);
|
|
|
+ uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
|
|
|
+ uasm_l_tlbl_goaround1(&l, p);
|
|
|
+ /* Reload the PTE value */
|
|
|
+ iPTE_LW(&p, K0, K1);
|
|
|
+ }
|
|
|
}
|
|
|
build_make_valid(&p, &r, K0, K1);
|
|
|
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
|
|
@@ -1442,23 +1839,35 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
|
|
|
* If the page is not _PAGE_VALID, RI or XI could not
|
|
|
* have triggered it. Skip the expensive test..
|
|
|
*/
|
|
|
- uasm_i_andi(&p, K0, K0, _PAGE_VALID);
|
|
|
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
|
|
|
+ label_tlbl_goaround2);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K0, _PAGE_VALID);
|
|
|
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
|
|
|
+ }
|
|
|
uasm_i_nop(&p);
|
|
|
|
|
|
uasm_i_tlbr(&p);
|
|
|
/* Examine entrylo 0 or 1 based on ptr. */
|
|
|
- uasm_i_andi(&p, K0, K1, sizeof(pte_t));
|
|
|
- uasm_i_beqz(&p, K0, 8);
|
|
|
-
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K1, sizeof(pte_t));
|
|
|
+ uasm_i_beqz(&p, K0, 8);
|
|
|
+ }
|
|
|
UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
|
|
|
UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
|
|
|
/*
|
|
|
* If the entryLo (now in K0) is valid (bit 1), RI or
|
|
|
* XI must have triggered it.
|
|
|
*/
|
|
|
- uasm_i_andi(&p, K0, K0, 2);
|
|
|
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
|
|
|
+ if (use_bbit_insns()) {
|
|
|
+ uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2);
|
|
|
+ } else {
|
|
|
+ uasm_i_andi(&p, K0, K0, 2);
|
|
|
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
|
|
|
+ }
|
|
|
/* Reload the PTE value */
|
|
|
iPTE_LW(&p, K0, K1);
|
|
|
|
|
@@ -1466,7 +1875,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
|
|
|
* We clobbered C0_PAGEMASK, restore it. On the other branch
|
|
|
* it is restored in build_huge_tlb_write_entry.
|
|
|
*/
|
|
|
- build_restore_pagemask(&p, &r, K0, label_nopage_tlbl);
|
|
|
+ build_restore_pagemask(&p, &r, K0, label_nopage_tlbl, 0);
|
|
|
|
|
|
uasm_l_tlbl_goaround2(&l, p);
|
|
|
}
|
|
@@ -1623,13 +2032,16 @@ void __cpuinit build_tlb_refill_handler(void)
|
|
|
break;
|
|
|
|
|
|
default:
|
|
|
- build_r4000_tlb_refill_handler();
|
|
|
if (!run_once) {
|
|
|
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
+ build_r4000_setup_pgd();
|
|
|
+#endif
|
|
|
build_r4000_tlb_load_handler();
|
|
|
build_r4000_tlb_store_handler();
|
|
|
build_r4000_tlb_modify_handler();
|
|
|
run_once++;
|
|
|
}
|
|
|
+ build_r4000_tlb_refill_handler();
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -1641,4 +2053,8 @@ void __cpuinit flush_tlb_handlers(void)
|
|
|
(unsigned long)handle_tlbs + sizeof(handle_tlbs));
|
|
|
local_flush_icache_range((unsigned long)handle_tlbm,
|
|
|
(unsigned long)handle_tlbm + sizeof(handle_tlbm));
|
|
|
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
|
|
|
+ local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
|
|
|
+ (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
|
|
|
+#endif
|
|
|
}
|