|
@@ -34,6 +34,7 @@ struct cpa_data {
|
|
unsigned long pfn;
|
|
unsigned long pfn;
|
|
unsigned force_split : 1;
|
|
unsigned force_split : 1;
|
|
int curpage;
|
|
int curpage;
|
|
|
|
+ struct page **pages;
|
|
};
|
|
};
|
|
|
|
|
|
/*
|
|
/*
|
|
@@ -46,6 +47,7 @@ static DEFINE_SPINLOCK(cpa_lock);
|
|
|
|
|
|
#define CPA_FLUSHTLB 1
|
|
#define CPA_FLUSHTLB 1
|
|
#define CPA_ARRAY 2
|
|
#define CPA_ARRAY 2
|
|
|
|
+#define CPA_PAGES_ARRAY 4
|
|
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
#ifdef CONFIG_PROC_FS
|
|
static unsigned long direct_pages_count[PG_LEVEL_NUM];
|
|
static unsigned long direct_pages_count[PG_LEVEL_NUM];
|
|
@@ -202,10 +204,10 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-static void cpa_flush_array(unsigned long *start, int numpages, int cache)
|
|
|
|
|
|
+static void cpa_flush_array(unsigned long *start, int numpages, int cache,
|
|
|
|
+ int in_flags, struct page **pages)
|
|
{
|
|
{
|
|
unsigned int i, level;
|
|
unsigned int i, level;
|
|
- unsigned long *addr;
|
|
|
|
|
|
|
|
BUG_ON(irqs_disabled());
|
|
BUG_ON(irqs_disabled());
|
|
|
|
|
|
@@ -226,14 +228,22 @@ static void cpa_flush_array(unsigned long *start, int numpages, int cache)
|
|
* will cause all other CPUs to flush the same
|
|
* will cause all other CPUs to flush the same
|
|
* cachelines:
|
|
* cachelines:
|
|
*/
|
|
*/
|
|
- for (i = 0, addr = start; i < numpages; i++, addr++) {
|
|
|
|
- pte_t *pte = lookup_address(*addr, &level);
|
|
|
|
|
|
+ for (i = 0; i < numpages; i++) {
|
|
|
|
+ unsigned long addr;
|
|
|
|
+ pte_t *pte;
|
|
|
|
+
|
|
|
|
+ if (in_flags & CPA_PAGES_ARRAY)
|
|
|
|
+ addr = (unsigned long)page_address(pages[i]);
|
|
|
|
+ else
|
|
|
|
+ addr = start[i];
|
|
|
|
+
|
|
|
|
+ pte = lookup_address(addr, &level);
|
|
|
|
|
|
/*
|
|
/*
|
|
* Only flush present addresses:
|
|
* Only flush present addresses:
|
|
*/
|
|
*/
|
|
if (pte && (pte_val(*pte) & _PAGE_PRESENT))
|
|
if (pte && (pte_val(*pte) & _PAGE_PRESENT))
|
|
- clflush_cache_range((void *) *addr, PAGE_SIZE);
|
|
|
|
|
|
+ clflush_cache_range((void *)addr, PAGE_SIZE);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -585,7 +595,9 @@ static int __change_page_attr(struct cpa_data *cpa, int primary)
|
|
unsigned int level;
|
|
unsigned int level;
|
|
pte_t *kpte, old_pte;
|
|
pte_t *kpte, old_pte;
|
|
|
|
|
|
- if (cpa->flags & CPA_ARRAY)
|
|
|
|
|
|
+ if (cpa->flags & CPA_PAGES_ARRAY)
|
|
|
|
+ address = (unsigned long)page_address(cpa->pages[cpa->curpage]);
|
|
|
|
+ else if (cpa->flags & CPA_ARRAY)
|
|
address = cpa->vaddr[cpa->curpage];
|
|
address = cpa->vaddr[cpa->curpage];
|
|
else
|
|
else
|
|
address = *cpa->vaddr;
|
|
address = *cpa->vaddr;
|
|
@@ -688,7 +700,9 @@ static int cpa_process_alias(struct cpa_data *cpa)
|
|
* No need to redo, when the primary call touched the direct
|
|
* No need to redo, when the primary call touched the direct
|
|
* mapping already:
|
|
* mapping already:
|
|
*/
|
|
*/
|
|
- if (cpa->flags & CPA_ARRAY)
|
|
|
|
|
|
+ if (cpa->flags & CPA_PAGES_ARRAY)
|
|
|
|
+ vaddr = (unsigned long)page_address(cpa->pages[cpa->curpage]);
|
|
|
|
+ else if (cpa->flags & CPA_ARRAY)
|
|
vaddr = cpa->vaddr[cpa->curpage];
|
|
vaddr = cpa->vaddr[cpa->curpage];
|
|
else
|
|
else
|
|
vaddr = *cpa->vaddr;
|
|
vaddr = *cpa->vaddr;
|
|
@@ -699,7 +713,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
|
|
alias_cpa = *cpa;
|
|
alias_cpa = *cpa;
|
|
temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
|
|
temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
|
|
alias_cpa.vaddr = &temp_cpa_vaddr;
|
|
alias_cpa.vaddr = &temp_cpa_vaddr;
|
|
- alias_cpa.flags &= ~CPA_ARRAY;
|
|
|
|
|
|
+ alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
|
|
|
|
|
|
|
|
|
|
ret = __change_page_attr_set_clr(&alias_cpa, 0);
|
|
ret = __change_page_attr_set_clr(&alias_cpa, 0);
|
|
@@ -725,7 +739,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
|
|
alias_cpa = *cpa;
|
|
alias_cpa = *cpa;
|
|
temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base;
|
|
temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base;
|
|
alias_cpa.vaddr = &temp_cpa_vaddr;
|
|
alias_cpa.vaddr = &temp_cpa_vaddr;
|
|
- alias_cpa.flags &= ~CPA_ARRAY;
|
|
|
|
|
|
+ alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
|
|
|
|
|
|
/*
|
|
/*
|
|
* The high mapping range is imprecise, so ignore the return value.
|
|
* The high mapping range is imprecise, so ignore the return value.
|
|
@@ -746,7 +760,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
|
|
*/
|
|
*/
|
|
cpa->numpages = numpages;
|
|
cpa->numpages = numpages;
|
|
/* for array changes, we can't use large page */
|
|
/* for array changes, we can't use large page */
|
|
- if (cpa->flags & CPA_ARRAY)
|
|
|
|
|
|
+ if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
|
|
cpa->numpages = 1;
|
|
cpa->numpages = 1;
|
|
|
|
|
|
if (!debug_pagealloc)
|
|
if (!debug_pagealloc)
|
|
@@ -770,7 +784,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
|
|
*/
|
|
*/
|
|
BUG_ON(cpa->numpages > numpages);
|
|
BUG_ON(cpa->numpages > numpages);
|
|
numpages -= cpa->numpages;
|
|
numpages -= cpa->numpages;
|
|
- if (cpa->flags & CPA_ARRAY)
|
|
|
|
|
|
+ if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
|
|
cpa->curpage++;
|
|
cpa->curpage++;
|
|
else
|
|
else
|
|
*cpa->vaddr += cpa->numpages * PAGE_SIZE;
|
|
*cpa->vaddr += cpa->numpages * PAGE_SIZE;
|
|
@@ -787,7 +801,8 @@ static inline int cache_attr(pgprot_t attr)
|
|
|
|
|
|
static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
pgprot_t mask_set, pgprot_t mask_clr,
|
|
pgprot_t mask_set, pgprot_t mask_clr,
|
|
- int force_split, int array)
|
|
|
|
|
|
+ int force_split, int in_flag,
|
|
|
|
+ struct page **pages)
|
|
{
|
|
{
|
|
struct cpa_data cpa;
|
|
struct cpa_data cpa;
|
|
int ret, cache, checkalias;
|
|
int ret, cache, checkalias;
|
|
@@ -802,15 +817,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
return 0;
|
|
return 0;
|
|
|
|
|
|
/* Ensure we are PAGE_SIZE aligned */
|
|
/* Ensure we are PAGE_SIZE aligned */
|
|
- if (!array) {
|
|
|
|
- if (*addr & ~PAGE_MASK) {
|
|
|
|
- *addr &= PAGE_MASK;
|
|
|
|
- /*
|
|
|
|
- * People should not be passing in unaligned addresses:
|
|
|
|
- */
|
|
|
|
- WARN_ON_ONCE(1);
|
|
|
|
- }
|
|
|
|
- } else {
|
|
|
|
|
|
+ if (in_flag & CPA_ARRAY) {
|
|
int i;
|
|
int i;
|
|
for (i = 0; i < numpages; i++) {
|
|
for (i = 0; i < numpages; i++) {
|
|
if (addr[i] & ~PAGE_MASK) {
|
|
if (addr[i] & ~PAGE_MASK) {
|
|
@@ -818,6 +825,18 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
WARN_ON_ONCE(1);
|
|
WARN_ON_ONCE(1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+ } else if (!(in_flag & CPA_PAGES_ARRAY)) {
|
|
|
|
+ /*
|
|
|
|
+ * in_flag of CPA_PAGES_ARRAY implies it is aligned.
|
|
|
|
+ * No need to cehck in that case
|
|
|
|
+ */
|
|
|
|
+ if (*addr & ~PAGE_MASK) {
|
|
|
|
+ *addr &= PAGE_MASK;
|
|
|
|
+ /*
|
|
|
|
+ * People should not be passing in unaligned addresses:
|
|
|
|
+ */
|
|
|
|
+ WARN_ON_ONCE(1);
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
/* Must avoid aliasing mappings in the highmem code */
|
|
/* Must avoid aliasing mappings in the highmem code */
|
|
@@ -833,6 +852,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
arch_flush_lazy_mmu_mode();
|
|
arch_flush_lazy_mmu_mode();
|
|
|
|
|
|
cpa.vaddr = addr;
|
|
cpa.vaddr = addr;
|
|
|
|
+ cpa.pages = pages;
|
|
cpa.numpages = numpages;
|
|
cpa.numpages = numpages;
|
|
cpa.mask_set = mask_set;
|
|
cpa.mask_set = mask_set;
|
|
cpa.mask_clr = mask_clr;
|
|
cpa.mask_clr = mask_clr;
|
|
@@ -840,8 +860,8 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
cpa.curpage = 0;
|
|
cpa.curpage = 0;
|
|
cpa.force_split = force_split;
|
|
cpa.force_split = force_split;
|
|
|
|
|
|
- if (array)
|
|
|
|
- cpa.flags |= CPA_ARRAY;
|
|
|
|
|
|
+ if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
|
|
|
|
+ cpa.flags |= in_flag;
|
|
|
|
|
|
/* No alias checking for _NX bit modifications */
|
|
/* No alias checking for _NX bit modifications */
|
|
checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
|
|
checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
|
|
@@ -867,9 +887,10 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
|
* wbindv):
|
|
* wbindv):
|
|
*/
|
|
*/
|
|
if (!ret && cpu_has_clflush) {
|
|
if (!ret && cpu_has_clflush) {
|
|
- if (cpa.flags & CPA_ARRAY)
|
|
|
|
- cpa_flush_array(addr, numpages, cache);
|
|
|
|
- else
|
|
|
|
|
|
+ if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
|
|
|
|
+ cpa_flush_array(addr, numpages, cache,
|
|
|
|
+ cpa.flags, pages);
|
|
|
|
+ } else
|
|
cpa_flush_range(*addr, numpages, cache);
|
|
cpa_flush_range(*addr, numpages, cache);
|
|
} else
|
|
} else
|
|
cpa_flush_all(cache);
|
|
cpa_flush_all(cache);
|
|
@@ -889,14 +910,28 @@ static inline int change_page_attr_set(unsigned long *addr, int numpages,
|
|
pgprot_t mask, int array)
|
|
pgprot_t mask, int array)
|
|
{
|
|
{
|
|
return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
|
|
return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
|
|
- array);
|
|
|
|
|
|
+ (array ? CPA_ARRAY : 0), NULL);
|
|
}
|
|
}
|
|
|
|
|
|
static inline int change_page_attr_clear(unsigned long *addr, int numpages,
|
|
static inline int change_page_attr_clear(unsigned long *addr, int numpages,
|
|
pgprot_t mask, int array)
|
|
pgprot_t mask, int array)
|
|
{
|
|
{
|
|
return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
|
|
return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
|
|
- array);
|
|
|
|
|
|
+ (array ? CPA_ARRAY : 0), NULL);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline int cpa_set_pages_array(struct page **pages, int numpages,
|
|
|
|
+ pgprot_t mask)
|
|
|
|
+{
|
|
|
|
+ return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
|
|
|
|
+ CPA_PAGES_ARRAY, pages);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline int cpa_clear_pages_array(struct page **pages, int numpages,
|
|
|
|
+ pgprot_t mask)
|
|
|
|
+{
|
|
|
|
+ return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
|
|
|
|
+ CPA_PAGES_ARRAY, pages);
|
|
}
|
|
}
|
|
|
|
|
|
int _set_memory_uc(unsigned long addr, int numpages)
|
|
int _set_memory_uc(unsigned long addr, int numpages)
|
|
@@ -1044,7 +1079,7 @@ int set_memory_np(unsigned long addr, int numpages)
|
|
int set_memory_4k(unsigned long addr, int numpages)
|
|
int set_memory_4k(unsigned long addr, int numpages)
|
|
{
|
|
{
|
|
return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
|
|
return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
|
|
- __pgprot(0), 1, 0);
|
|
|
|
|
|
+ __pgprot(0), 1, 0, NULL);
|
|
}
|
|
}
|
|
|
|
|
|
int set_pages_uc(struct page *page, int numpages)
|
|
int set_pages_uc(struct page *page, int numpages)
|
|
@@ -1055,6 +1090,35 @@ int set_pages_uc(struct page *page, int numpages)
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(set_pages_uc);
|
|
EXPORT_SYMBOL(set_pages_uc);
|
|
|
|
|
|
|
|
+int set_pages_array_uc(struct page **pages, int addrinarray)
|
|
|
|
+{
|
|
|
|
+ unsigned long start;
|
|
|
|
+ unsigned long end;
|
|
|
|
+ int i;
|
|
|
|
+ int free_idx;
|
|
|
|
+
|
|
|
|
+ for (i = 0; i < addrinarray; i++) {
|
|
|
|
+ start = (unsigned long)page_address(pages[i]);
|
|
|
|
+ end = start + PAGE_SIZE;
|
|
|
|
+ if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL))
|
|
|
|
+ goto err_out;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (cpa_set_pages_array(pages, addrinarray,
|
|
|
|
+ __pgprot(_PAGE_CACHE_UC_MINUS)) == 0) {
|
|
|
|
+ return 0; /* Success */
|
|
|
|
+ }
|
|
|
|
+err_out:
|
|
|
|
+ free_idx = i;
|
|
|
|
+ for (i = 0; i < free_idx; i++) {
|
|
|
|
+ start = (unsigned long)page_address(pages[i]);
|
|
|
|
+ end = start + PAGE_SIZE;
|
|
|
|
+ free_memtype(start, end);
|
|
|
|
+ }
|
|
|
|
+ return -EINVAL;
|
|
|
|
+}
|
|
|
|
+EXPORT_SYMBOL(set_pages_array_uc);
|
|
|
|
+
|
|
int set_pages_wb(struct page *page, int numpages)
|
|
int set_pages_wb(struct page *page, int numpages)
|
|
{
|
|
{
|
|
unsigned long addr = (unsigned long)page_address(page);
|
|
unsigned long addr = (unsigned long)page_address(page);
|
|
@@ -1063,6 +1127,26 @@ int set_pages_wb(struct page *page, int numpages)
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(set_pages_wb);
|
|
EXPORT_SYMBOL(set_pages_wb);
|
|
|
|
|
|
|
|
+int set_pages_array_wb(struct page **pages, int addrinarray)
|
|
|
|
+{
|
|
|
|
+ int retval;
|
|
|
|
+ unsigned long start;
|
|
|
|
+ unsigned long end;
|
|
|
|
+ int i;
|
|
|
|
+
|
|
|
|
+ retval = cpa_clear_pages_array(pages, addrinarray,
|
|
|
|
+ __pgprot(_PAGE_CACHE_MASK));
|
|
|
|
+
|
|
|
|
+ for (i = 0; i < addrinarray; i++) {
|
|
|
|
+ start = (unsigned long)page_address(pages[i]);
|
|
|
|
+ end = start + PAGE_SIZE;
|
|
|
|
+ free_memtype(start, end);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return retval;
|
|
|
|
+}
|
|
|
|
+EXPORT_SYMBOL(set_pages_array_wb);
|
|
|
|
+
|
|
int set_pages_x(struct page *page, int numpages)
|
|
int set_pages_x(struct page *page, int numpages)
|
|
{
|
|
{
|
|
unsigned long addr = (unsigned long)page_address(page);
|
|
unsigned long addr = (unsigned long)page_address(page);
|