|
@@ -263,6 +263,24 @@ static int shmem_radix_tree_replace(struct address_space *mapping,
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * Sometimes, before we decide whether to proceed or to fail, we must check
|
|
|
+ * that an entry was not already brought back from swap by a racing thread.
|
|
|
+ *
|
|
|
+ * Checking page is not enough: by the time a SwapCache page is locked, it
|
|
|
+ * might be reused, and again be SwapCache, using the same swap as before.
|
|
|
+ */
|
|
|
+static bool shmem_confirm_swap(struct address_space *mapping,
|
|
|
+ pgoff_t index, swp_entry_t swap)
|
|
|
+{
|
|
|
+ void *item;
|
|
|
+
|
|
|
+ rcu_read_lock();
|
|
|
+ item = radix_tree_lookup(&mapping->page_tree, index);
|
|
|
+ rcu_read_unlock();
|
|
|
+ return item == swp_to_radix_entry(swap);
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Like add_to_page_cache_locked, but error if expected item has gone.
|
|
|
*/
|
|
@@ -270,40 +288,31 @@ static int shmem_add_to_page_cache(struct page *page,
|
|
|
struct address_space *mapping,
|
|
|
pgoff_t index, gfp_t gfp, void *expected)
|
|
|
{
|
|
|
- int error = 0;
|
|
|
+ int error;
|
|
|
|
|
|
VM_BUG_ON(!PageLocked(page));
|
|
|
VM_BUG_ON(!PageSwapBacked(page));
|
|
|
|
|
|
+ page_cache_get(page);
|
|
|
+ page->mapping = mapping;
|
|
|
+ page->index = index;
|
|
|
+
|
|
|
+ spin_lock_irq(&mapping->tree_lock);
|
|
|
if (!expected)
|
|
|
- error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
|
|
|
+ error = radix_tree_insert(&mapping->page_tree, index, page);
|
|
|
+ else
|
|
|
+ error = shmem_radix_tree_replace(mapping, index, expected,
|
|
|
+ page);
|
|
|
if (!error) {
|
|
|
- page_cache_get(page);
|
|
|
- page->mapping = mapping;
|
|
|
- page->index = index;
|
|
|
-
|
|
|
- spin_lock_irq(&mapping->tree_lock);
|
|
|
- if (!expected)
|
|
|
- error = radix_tree_insert(&mapping->page_tree,
|
|
|
- index, page);
|
|
|
- else
|
|
|
- error = shmem_radix_tree_replace(mapping, index,
|
|
|
- expected, page);
|
|
|
- if (!error) {
|
|
|
- mapping->nrpages++;
|
|
|
- __inc_zone_page_state(page, NR_FILE_PAGES);
|
|
|
- __inc_zone_page_state(page, NR_SHMEM);
|
|
|
- spin_unlock_irq(&mapping->tree_lock);
|
|
|
- } else {
|
|
|
- page->mapping = NULL;
|
|
|
- spin_unlock_irq(&mapping->tree_lock);
|
|
|
- page_cache_release(page);
|
|
|
- }
|
|
|
- if (!expected)
|
|
|
- radix_tree_preload_end();
|
|
|
+ mapping->nrpages++;
|
|
|
+ __inc_zone_page_state(page, NR_FILE_PAGES);
|
|
|
+ __inc_zone_page_state(page, NR_SHMEM);
|
|
|
+ spin_unlock_irq(&mapping->tree_lock);
|
|
|
+ } else {
|
|
|
+ page->mapping = NULL;
|
|
|
+ spin_unlock_irq(&mapping->tree_lock);
|
|
|
+ page_cache_release(page);
|
|
|
}
|
|
|
- if (error)
|
|
|
- mem_cgroup_uncharge_cache_page(page);
|
|
|
return error;
|
|
|
}
|
|
|
|
|
@@ -1124,9 +1133,9 @@ repeat:
|
|
|
/* We have to do this with page locked to prevent races */
|
|
|
lock_page(page);
|
|
|
if (!PageSwapCache(page) || page_private(page) != swap.val ||
|
|
|
- page->mapping) {
|
|
|
+ !shmem_confirm_swap(mapping, index, swap)) {
|
|
|
error = -EEXIST; /* try again */
|
|
|
- goto failed;
|
|
|
+ goto unlock;
|
|
|
}
|
|
|
if (!PageUptodate(page)) {
|
|
|
error = -EIO;
|
|
@@ -1142,9 +1151,12 @@ repeat:
|
|
|
|
|
|
error = mem_cgroup_cache_charge(page, current->mm,
|
|
|
gfp & GFP_RECLAIM_MASK);
|
|
|
- if (!error)
|
|
|
+ if (!error) {
|
|
|
error = shmem_add_to_page_cache(page, mapping, index,
|
|
|
gfp, swp_to_radix_entry(swap));
|
|
|
+ /* We already confirmed swap, and make no allocation */
|
|
|
+ VM_BUG_ON(error);
|
|
|
+ }
|
|
|
if (error)
|
|
|
goto failed;
|
|
|
|
|
@@ -1181,11 +1193,18 @@ repeat:
|
|
|
__set_page_locked(page);
|
|
|
error = mem_cgroup_cache_charge(page, current->mm,
|
|
|
gfp & GFP_RECLAIM_MASK);
|
|
|
- if (!error)
|
|
|
- error = shmem_add_to_page_cache(page, mapping, index,
|
|
|
- gfp, NULL);
|
|
|
if (error)
|
|
|
goto decused;
|
|
|
+ error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
|
|
|
+ if (!error) {
|
|
|
+ error = shmem_add_to_page_cache(page, mapping, index,
|
|
|
+ gfp, NULL);
|
|
|
+ radix_tree_preload_end();
|
|
|
+ }
|
|
|
+ if (error) {
|
|
|
+ mem_cgroup_uncharge_cache_page(page);
|
|
|
+ goto decused;
|
|
|
+ }
|
|
|
lru_cache_add_anon(page);
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
@@ -1245,14 +1264,10 @@ decused:
|
|
|
unacct:
|
|
|
shmem_unacct_blocks(info->flags, 1);
|
|
|
failed:
|
|
|
- if (swap.val && error != -EINVAL) {
|
|
|
- struct page *test = find_get_page(mapping, index);
|
|
|
- if (test && !radix_tree_exceptional_entry(test))
|
|
|
- page_cache_release(test);
|
|
|
- /* Have another try if the entry has changed */
|
|
|
- if (test != swp_to_radix_entry(swap))
|
|
|
- error = -EEXIST;
|
|
|
- }
|
|
|
+ if (swap.val && error != -EINVAL &&
|
|
|
+ !shmem_confirm_swap(mapping, index, swap))
|
|
|
+ error = -EEXIST;
|
|
|
+unlock:
|
|
|
if (page) {
|
|
|
unlock_page(page);
|
|
|
page_cache_release(page);
|
|
@@ -1264,7 +1279,7 @@ failed:
|
|
|
spin_unlock(&info->lock);
|
|
|
goto repeat;
|
|
|
}
|
|
|
- if (error == -EEXIST)
|
|
|
+ if (error == -EEXIST) /* from above or from radix_tree_insert */
|
|
|
goto repeat;
|
|
|
return error;
|
|
|
}
|
|
@@ -1692,98 +1707,6 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
|
|
|
return error;
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
|
|
|
- */
|
|
|
-static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
|
|
|
- pgoff_t index, pgoff_t end, int origin)
|
|
|
-{
|
|
|
- struct page *page;
|
|
|
- struct pagevec pvec;
|
|
|
- pgoff_t indices[PAGEVEC_SIZE];
|
|
|
- bool done = false;
|
|
|
- int i;
|
|
|
-
|
|
|
- pagevec_init(&pvec, 0);
|
|
|
- pvec.nr = 1; /* start small: we may be there already */
|
|
|
- while (!done) {
|
|
|
- pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
|
|
|
- pvec.nr, pvec.pages, indices);
|
|
|
- if (!pvec.nr) {
|
|
|
- if (origin == SEEK_DATA)
|
|
|
- index = end;
|
|
|
- break;
|
|
|
- }
|
|
|
- for (i = 0; i < pvec.nr; i++, index++) {
|
|
|
- if (index < indices[i]) {
|
|
|
- if (origin == SEEK_HOLE) {
|
|
|
- done = true;
|
|
|
- break;
|
|
|
- }
|
|
|
- index = indices[i];
|
|
|
- }
|
|
|
- page = pvec.pages[i];
|
|
|
- if (page && !radix_tree_exceptional_entry(page)) {
|
|
|
- if (!PageUptodate(page))
|
|
|
- page = NULL;
|
|
|
- }
|
|
|
- if (index >= end ||
|
|
|
- (page && origin == SEEK_DATA) ||
|
|
|
- (!page && origin == SEEK_HOLE)) {
|
|
|
- done = true;
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- shmem_deswap_pagevec(&pvec);
|
|
|
- pagevec_release(&pvec);
|
|
|
- pvec.nr = PAGEVEC_SIZE;
|
|
|
- cond_resched();
|
|
|
- }
|
|
|
- return index;
|
|
|
-}
|
|
|
-
|
|
|
-static loff_t shmem_file_llseek(struct file *file, loff_t offset, int origin)
|
|
|
-{
|
|
|
- struct address_space *mapping;
|
|
|
- struct inode *inode;
|
|
|
- pgoff_t start, end;
|
|
|
- loff_t new_offset;
|
|
|
-
|
|
|
- if (origin != SEEK_DATA && origin != SEEK_HOLE)
|
|
|
- return generic_file_llseek_size(file, offset, origin,
|
|
|
- MAX_LFS_FILESIZE);
|
|
|
- mapping = file->f_mapping;
|
|
|
- inode = mapping->host;
|
|
|
- mutex_lock(&inode->i_mutex);
|
|
|
- /* We're holding i_mutex so we can access i_size directly */
|
|
|
-
|
|
|
- if (offset < 0)
|
|
|
- offset = -EINVAL;
|
|
|
- else if (offset >= inode->i_size)
|
|
|
- offset = -ENXIO;
|
|
|
- else {
|
|
|
- start = offset >> PAGE_CACHE_SHIFT;
|
|
|
- end = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
|
|
|
- new_offset = shmem_seek_hole_data(mapping, start, end, origin);
|
|
|
- new_offset <<= PAGE_CACHE_SHIFT;
|
|
|
- if (new_offset > offset) {
|
|
|
- if (new_offset < inode->i_size)
|
|
|
- offset = new_offset;
|
|
|
- else if (origin == SEEK_DATA)
|
|
|
- offset = -ENXIO;
|
|
|
- else
|
|
|
- offset = inode->i_size;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if (offset >= 0 && offset != file->f_pos) {
|
|
|
- file->f_pos = offset;
|
|
|
- file->f_version = 0;
|
|
|
- }
|
|
|
- mutex_unlock(&inode->i_mutex);
|
|
|
- return offset;
|
|
|
-}
|
|
|
-
|
|
|
static long shmem_fallocate(struct file *file, int mode, loff_t offset,
|
|
|
loff_t len)
|
|
|
{
|
|
@@ -2787,7 +2710,7 @@ static const struct address_space_operations shmem_aops = {
|
|
|
static const struct file_operations shmem_file_operations = {
|
|
|
.mmap = shmem_mmap,
|
|
|
#ifdef CONFIG_TMPFS
|
|
|
- .llseek = shmem_file_llseek,
|
|
|
+ .llseek = generic_file_llseek,
|
|
|
.read = do_sync_read,
|
|
|
.write = do_sync_write,
|
|
|
.aio_read = shmem_file_aio_read,
|