|
@@ -787,6 +787,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
struct inode *inode;
|
|
|
char *data;
|
|
|
char *bitmap;
|
|
|
+ struct ext4_group_info *grinfo;
|
|
|
|
|
|
mb_debug(1, "init page %lu\n", page->index);
|
|
|
|
|
@@ -819,6 +820,18 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
if (first_group + i >= ngroups)
|
|
|
break;
|
|
|
|
|
|
+ grinfo = ext4_get_group_info(sb, first_group + i);
|
|
|
+ /*
|
|
|
+ * If page is uptodate then we came here after online resize
|
|
|
+ * which added some new uninitialized group info structs, so
|
|
|
+ * we must skip all initialized uptodate buddies on the page,
|
|
|
+ * which may be currently in use by an allocating task.
|
|
|
+ */
|
|
|
+ if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) {
|
|
|
+ bh[i] = NULL;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
err = -EIO;
|
|
|
desc = ext4_get_group_desc(sb, first_group + i, NULL);
|
|
|
if (desc == NULL)
|
|
@@ -871,26 +884,28 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
}
|
|
|
|
|
|
/* wait for I/O completion */
|
|
|
- for (i = 0; i < groups_per_page && bh[i]; i++)
|
|
|
- wait_on_buffer(bh[i]);
|
|
|
+ for (i = 0; i < groups_per_page; i++)
|
|
|
+ if (bh[i])
|
|
|
+ wait_on_buffer(bh[i]);
|
|
|
|
|
|
err = -EIO;
|
|
|
- for (i = 0; i < groups_per_page && bh[i]; i++)
|
|
|
- if (!buffer_uptodate(bh[i]))
|
|
|
+ for (i = 0; i < groups_per_page; i++)
|
|
|
+ if (bh[i] && !buffer_uptodate(bh[i]))
|
|
|
goto out;
|
|
|
|
|
|
err = 0;
|
|
|
first_block = page->index * blocks_per_page;
|
|
|
- /* init the page */
|
|
|
- memset(page_address(page), 0xff, PAGE_CACHE_SIZE);
|
|
|
for (i = 0; i < blocks_per_page; i++) {
|
|
|
int group;
|
|
|
- struct ext4_group_info *grinfo;
|
|
|
|
|
|
group = (first_block + i) >> 1;
|
|
|
if (group >= ngroups)
|
|
|
break;
|
|
|
|
|
|
+ if (!bh[group - first_group])
|
|
|
+ /* skip initialized uptodate buddy */
|
|
|
+ continue;
|
|
|
+
|
|
|
/*
|
|
|
* data carry information regarding this
|
|
|
* particular group in the format specified
|
|
@@ -919,6 +934,8 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
* incore got set to the group block bitmap below
|
|
|
*/
|
|
|
ext4_lock_group(sb, group);
|
|
|
+ /* init the buddy */
|
|
|
+ memset(data, 0xff, blocksize);
|
|
|
ext4_mb_generate_buddy(sb, data, incore, group);
|
|
|
ext4_unlock_group(sb, group);
|
|
|
incore = NULL;
|
|
@@ -948,7 +965,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
|
|
|
out:
|
|
|
if (bh) {
|
|
|
- for (i = 0; i < groups_per_page && bh[i]; i++)
|
|
|
+ for (i = 0; i < groups_per_page; i++)
|
|
|
brelse(bh[i]);
|
|
|
if (bh != &bhs)
|
|
|
kfree(bh);
|
|
@@ -957,22 +974,21 @@ out:
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * lock the group_info alloc_sem of all the groups
|
|
|
- * belonging to the same buddy cache page. This
|
|
|
- * make sure other parallel operation on the buddy
|
|
|
- * cache doesn't happen whild holding the buddy cache
|
|
|
- * lock
|
|
|
+ * Lock the buddy and bitmap pages. This make sure other parallel init_group
|
|
|
+ * on the same buddy page doesn't happen whild holding the buddy page lock.
|
|
|
+ * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
|
|
|
+ * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
|
|
|
*/
|
|
|
-static int ext4_mb_get_buddy_cache_lock(struct super_block *sb,
|
|
|
- ext4_group_t group)
|
|
|
+static int ext4_mb_get_buddy_page_lock(struct super_block *sb,
|
|
|
+ ext4_group_t group, struct ext4_buddy *e4b)
|
|
|
{
|
|
|
- int i;
|
|
|
- int block, pnum;
|
|
|
+ struct inode *inode = EXT4_SB(sb)->s_buddy_cache;
|
|
|
+ int block, pnum, poff;
|
|
|
int blocks_per_page;
|
|
|
- int groups_per_page;
|
|
|
- ext4_group_t ngroups = ext4_get_groups_count(sb);
|
|
|
- ext4_group_t first_group;
|
|
|
- struct ext4_group_info *grp;
|
|
|
+ struct page *page;
|
|
|
+
|
|
|
+ e4b->bd_buddy_page = NULL;
|
|
|
+ e4b->bd_bitmap_page = NULL;
|
|
|
|
|
|
blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
/*
|
|
@@ -982,57 +998,40 @@ static int ext4_mb_get_buddy_cache_lock(struct super_block *sb,
|
|
|
*/
|
|
|
block = group * 2;
|
|
|
pnum = block / blocks_per_page;
|
|
|
- first_group = pnum * blocks_per_page / 2;
|
|
|
-
|
|
|
- groups_per_page = blocks_per_page >> 1;
|
|
|
- if (groups_per_page == 0)
|
|
|
- groups_per_page = 1;
|
|
|
- /* read all groups the page covers into the cache */
|
|
|
- for (i = 0; i < groups_per_page; i++) {
|
|
|
+ poff = block % blocks_per_page;
|
|
|
+ page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
+ if (!page)
|
|
|
+ return -EIO;
|
|
|
+ BUG_ON(page->mapping != inode->i_mapping);
|
|
|
+ e4b->bd_bitmap_page = page;
|
|
|
+ e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
|
|
|
|
|
|
- if ((first_group + i) >= ngroups)
|
|
|
- break;
|
|
|
- grp = ext4_get_group_info(sb, first_group + i);
|
|
|
- /* take all groups write allocation
|
|
|
- * semaphore. This make sure there is
|
|
|
- * no block allocation going on in any
|
|
|
- * of that groups
|
|
|
- */
|
|
|
- down_write_nested(&grp->alloc_sem, i);
|
|
|
+ if (blocks_per_page >= 2) {
|
|
|
+ /* buddy and bitmap are on the same page */
|
|
|
+ return 0;
|
|
|
}
|
|
|
- return i;
|
|
|
+
|
|
|
+ block++;
|
|
|
+ pnum = block / blocks_per_page;
|
|
|
+ poff = block % blocks_per_page;
|
|
|
+ page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
+ if (!page)
|
|
|
+ return -EIO;
|
|
|
+ BUG_ON(page->mapping != inode->i_mapping);
|
|
|
+ e4b->bd_buddy_page = page;
|
|
|
+ return 0;
|
|
|
}
|
|
|
|
|
|
-static void ext4_mb_put_buddy_cache_lock(struct super_block *sb,
|
|
|
- ext4_group_t group, int locked_group)
|
|
|
+static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
|
|
|
{
|
|
|
- int i;
|
|
|
- int block, pnum;
|
|
|
- int blocks_per_page;
|
|
|
- ext4_group_t first_group;
|
|
|
- struct ext4_group_info *grp;
|
|
|
-
|
|
|
- blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
- /*
|
|
|
- * the buddy cache inode stores the block bitmap
|
|
|
- * and buddy information in consecutive blocks.
|
|
|
- * So for each group we need two blocks.
|
|
|
- */
|
|
|
- block = group * 2;
|
|
|
- pnum = block / blocks_per_page;
|
|
|
- first_group = pnum * blocks_per_page / 2;
|
|
|
- /* release locks on all the groups */
|
|
|
- for (i = 0; i < locked_group; i++) {
|
|
|
-
|
|
|
- grp = ext4_get_group_info(sb, first_group + i);
|
|
|
- /* take all groups write allocation
|
|
|
- * semaphore. This make sure there is
|
|
|
- * no block allocation going on in any
|
|
|
- * of that groups
|
|
|
- */
|
|
|
- up_write(&grp->alloc_sem);
|
|
|
+ if (e4b->bd_bitmap_page) {
|
|
|
+ unlock_page(e4b->bd_bitmap_page);
|
|
|
+ page_cache_release(e4b->bd_bitmap_page);
|
|
|
+ }
|
|
|
+ if (e4b->bd_buddy_page) {
|
|
|
+ unlock_page(e4b->bd_buddy_page);
|
|
|
+ page_cache_release(e4b->bd_buddy_page);
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1044,93 +1043,60 @@ static noinline_for_stack
|
|
|
int ext4_mb_init_group(struct super_block *sb, ext4_group_t group)
|
|
|
{
|
|
|
|
|
|
- int ret = 0;
|
|
|
- void *bitmap;
|
|
|
- int blocks_per_page;
|
|
|
- int block, pnum, poff;
|
|
|
- int num_grp_locked = 0;
|
|
|
struct ext4_group_info *this_grp;
|
|
|
- struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
- struct inode *inode = sbi->s_buddy_cache;
|
|
|
- struct page *page = NULL, *bitmap_page = NULL;
|
|
|
+ struct ext4_buddy e4b;
|
|
|
+ struct page *page;
|
|
|
+ int ret = 0;
|
|
|
|
|
|
mb_debug(1, "init group %u\n", group);
|
|
|
- blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
this_grp = ext4_get_group_info(sb, group);
|
|
|
/*
|
|
|
* This ensures that we don't reinit the buddy cache
|
|
|
* page which map to the group from which we are already
|
|
|
* allocating. If we are looking at the buddy cache we would
|
|
|
* have taken a reference using ext4_mb_load_buddy and that
|
|
|
- * would have taken the alloc_sem lock.
|
|
|
+ * would have pinned buddy page to page cache.
|
|
|
*/
|
|
|
- num_grp_locked = ext4_mb_get_buddy_cache_lock(sb, group);
|
|
|
- if (!EXT4_MB_GRP_NEED_INIT(this_grp)) {
|
|
|
+ ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b);
|
|
|
+ if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) {
|
|
|
/*
|
|
|
* somebody initialized the group
|
|
|
* return without doing anything
|
|
|
*/
|
|
|
- ret = 0;
|
|
|
goto err;
|
|
|
}
|
|
|
- /*
|
|
|
- * the buddy cache inode stores the block bitmap
|
|
|
- * and buddy information in consecutive blocks.
|
|
|
- * So for each group we need two blocks.
|
|
|
- */
|
|
|
- block = group * 2;
|
|
|
- pnum = block / blocks_per_page;
|
|
|
- poff = block % blocks_per_page;
|
|
|
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
- if (page) {
|
|
|
- BUG_ON(page->mapping != inode->i_mapping);
|
|
|
- ret = ext4_mb_init_cache(page, NULL);
|
|
|
- if (ret) {
|
|
|
- unlock_page(page);
|
|
|
- goto err;
|
|
|
- }
|
|
|
- unlock_page(page);
|
|
|
- }
|
|
|
- if (page == NULL || !PageUptodate(page)) {
|
|
|
+
|
|
|
+ page = e4b.bd_bitmap_page;
|
|
|
+ ret = ext4_mb_init_cache(page, NULL);
|
|
|
+ if (ret)
|
|
|
+ goto err;
|
|
|
+ if (!PageUptodate(page)) {
|
|
|
ret = -EIO;
|
|
|
goto err;
|
|
|
}
|
|
|
mark_page_accessed(page);
|
|
|
- bitmap_page = page;
|
|
|
- bitmap = page_address(page) + (poff * sb->s_blocksize);
|
|
|
|
|
|
- /* init buddy cache */
|
|
|
- block++;
|
|
|
- pnum = block / blocks_per_page;
|
|
|
- poff = block % blocks_per_page;
|
|
|
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
- if (page == bitmap_page) {
|
|
|
+ if (e4b.bd_buddy_page == NULL) {
|
|
|
/*
|
|
|
* If both the bitmap and buddy are in
|
|
|
* the same page we don't need to force
|
|
|
* init the buddy
|
|
|
*/
|
|
|
- unlock_page(page);
|
|
|
- } else if (page) {
|
|
|
- BUG_ON(page->mapping != inode->i_mapping);
|
|
|
- ret = ext4_mb_init_cache(page, bitmap);
|
|
|
- if (ret) {
|
|
|
- unlock_page(page);
|
|
|
- goto err;
|
|
|
- }
|
|
|
- unlock_page(page);
|
|
|
+ ret = 0;
|
|
|
+ goto err;
|
|
|
}
|
|
|
- if (page == NULL || !PageUptodate(page)) {
|
|
|
+ /* init buddy cache */
|
|
|
+ page = e4b.bd_buddy_page;
|
|
|
+ ret = ext4_mb_init_cache(page, e4b.bd_bitmap);
|
|
|
+ if (ret)
|
|
|
+ goto err;
|
|
|
+ if (!PageUptodate(page)) {
|
|
|
ret = -EIO;
|
|
|
goto err;
|
|
|
}
|
|
|
mark_page_accessed(page);
|
|
|
err:
|
|
|
- ext4_mb_put_buddy_cache_lock(sb, group, num_grp_locked);
|
|
|
- if (bitmap_page)
|
|
|
- page_cache_release(bitmap_page);
|
|
|
- if (page)
|
|
|
- page_cache_release(page);
|
|
|
+ ext4_mb_put_buddy_page_lock(&e4b);
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
@@ -1164,24 +1130,8 @@ ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
|
|
|
e4b->bd_group = group;
|
|
|
e4b->bd_buddy_page = NULL;
|
|
|
e4b->bd_bitmap_page = NULL;
|
|
|
- e4b->alloc_semp = &grp->alloc_sem;
|
|
|
-
|
|
|
- /* Take the read lock on the group alloc
|
|
|
- * sem. This would make sure a parallel
|
|
|
- * ext4_mb_init_group happening on other
|
|
|
- * groups mapped by the page is blocked
|
|
|
- * till we are done with allocation
|
|
|
- */
|
|
|
-repeat_load_buddy:
|
|
|
- down_read(e4b->alloc_semp);
|
|
|
|
|
|
if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
|
|
|
- /* we need to check for group need init flag
|
|
|
- * with alloc_semp held so that we can be sure
|
|
|
- * that new blocks didn't get added to the group
|
|
|
- * when we are loading the buddy cache
|
|
|
- */
|
|
|
- up_read(e4b->alloc_semp);
|
|
|
/*
|
|
|
* we need full data about the group
|
|
|
* to make a good selection
|
|
@@ -1189,7 +1139,6 @@ repeat_load_buddy:
|
|
|
ret = ext4_mb_init_group(sb, group);
|
|
|
if (ret)
|
|
|
return ret;
|
|
|
- goto repeat_load_buddy;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1273,15 +1222,14 @@ repeat_load_buddy:
|
|
|
return 0;
|
|
|
|
|
|
err:
|
|
|
+ if (page)
|
|
|
+ page_cache_release(page);
|
|
|
if (e4b->bd_bitmap_page)
|
|
|
page_cache_release(e4b->bd_bitmap_page);
|
|
|
if (e4b->bd_buddy_page)
|
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
|
e4b->bd_buddy = NULL;
|
|
|
e4b->bd_bitmap = NULL;
|
|
|
-
|
|
|
- /* Done with the buddy cache */
|
|
|
- up_read(e4b->alloc_semp);
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
@@ -1291,9 +1239,6 @@ static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
|
|
|
page_cache_release(e4b->bd_bitmap_page);
|
|
|
if (e4b->bd_buddy_page)
|
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
|
- /* Done with the buddy cache */
|
|
|
- if (e4b->alloc_semp)
|
|
|
- up_read(e4b->alloc_semp);
|
|
|
}
|
|
|
|
|
|
|
|
@@ -1606,9 +1551,6 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
|
|
|
get_page(ac->ac_bitmap_page);
|
|
|
ac->ac_buddy_page = e4b->bd_buddy_page;
|
|
|
get_page(ac->ac_buddy_page);
|
|
|
- /* on allocation we use ac to track the held semaphore */
|
|
|
- ac->alloc_semp = e4b->alloc_semp;
|
|
|
- e4b->alloc_semp = NULL;
|
|
|
/* store last allocated for subsequent stream allocation */
|
|
|
if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
|
|
|
spin_lock(&sbi->s_md_lock);
|
|
@@ -2659,7 +2601,7 @@ static void release_blocks_on_commit(journal_t *journal, transaction_t *txn)
|
|
|
struct super_block *sb = journal->j_private;
|
|
|
struct ext4_buddy e4b;
|
|
|
struct ext4_group_info *db;
|
|
|
- int err, ret, count = 0, count2 = 0;
|
|
|
+ int err, count = 0, count2 = 0;
|
|
|
struct ext4_free_data *entry;
|
|
|
struct list_head *l, *ltmp;
|
|
|
|
|
@@ -2669,15 +2611,9 @@ static void release_blocks_on_commit(journal_t *journal, transaction_t *txn)
|
|
|
mb_debug(1, "gonna free %u blocks in group %u (0x%p):",
|
|
|
entry->count, entry->group, entry);
|
|
|
|
|
|
- if (test_opt(sb, DISCARD)) {
|
|
|
- ret = ext4_issue_discard(sb, entry->group,
|
|
|
- entry->start_blk, entry->count);
|
|
|
- if (unlikely(ret == -EOPNOTSUPP)) {
|
|
|
- ext4_warning(sb, "discard not supported, "
|
|
|
- "disabling");
|
|
|
- clear_opt(sb, DISCARD);
|
|
|
- }
|
|
|
- }
|
|
|
+ if (test_opt(sb, DISCARD))
|
|
|
+ ext4_issue_discard(sb, entry->group,
|
|
|
+ entry->start_blk, entry->count);
|
|
|
|
|
|
err = ext4_mb_load_buddy(sb, entry->group, &e4b);
|
|
|
/* we expect to find existing buddy because it's pinned */
|
|
@@ -4226,15 +4162,12 @@ static int ext4_mb_release_context(struct ext4_allocation_context *ac)
|
|
|
spin_unlock(&pa->pa_lock);
|
|
|
}
|
|
|
}
|
|
|
- if (ac->alloc_semp)
|
|
|
- up_read(ac->alloc_semp);
|
|
|
if (pa) {
|
|
|
/*
|
|
|
* We want to add the pa to the right bucket.
|
|
|
* Remove it from the list and while adding
|
|
|
* make sure the list to which we are adding
|
|
|
- * doesn't grow big. We need to release
|
|
|
- * alloc_semp before calling ext4_mb_add_n_trim()
|
|
|
+ * doesn't grow big.
|
|
|
*/
|
|
|
if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) {
|
|
|
spin_lock(pa->pa_obj_lock);
|
|
@@ -4303,7 +4236,9 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
|
|
|
* there is enough free blocks to do block allocation
|
|
|
* and verify allocation doesn't exceed the quota limits.
|
|
|
*/
|
|
|
- while (ar->len && ext4_claim_free_blocks(sbi, ar->len)) {
|
|
|
+ while (ar->len &&
|
|
|
+ ext4_claim_free_blocks(sbi, ar->len, ar->flags)) {
|
|
|
+
|
|
|
/* let others to free the space */
|
|
|
yield();
|
|
|
ar->len = ar->len >> 1;
|
|
@@ -4313,9 +4248,15 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
|
|
|
return 0;
|
|
|
}
|
|
|
reserv_blks = ar->len;
|
|
|
- while (ar->len && dquot_alloc_block(ar->inode, ar->len)) {
|
|
|
- ar->flags |= EXT4_MB_HINT_NOPREALLOC;
|
|
|
- ar->len--;
|
|
|
+ if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {
|
|
|
+ dquot_alloc_block_nofail(ar->inode, ar->len);
|
|
|
+ } else {
|
|
|
+ while (ar->len &&
|
|
|
+ dquot_alloc_block(ar->inode, ar->len)) {
|
|
|
+
|
|
|
+ ar->flags |= EXT4_MB_HINT_NOPREALLOC;
|
|
|
+ ar->len--;
|
|
|
+ }
|
|
|
}
|
|
|
inquota = ar->len;
|
|
|
if (ar->len == 0) {
|
|
@@ -4703,6 +4644,127 @@ error_return:
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * ext4_add_groupblocks() -- Add given blocks to an existing group
|
|
|
+ * @handle: handle to this transaction
|
|
|
+ * @sb: super block
|
|
|
+ * @block: start physcial block to add to the block group
|
|
|
+ * @count: number of blocks to free
|
|
|
+ *
|
|
|
+ * This marks the blocks as free in the bitmap and buddy.
|
|
|
+ */
|
|
|
+void ext4_add_groupblocks(handle_t *handle, struct super_block *sb,
|
|
|
+ ext4_fsblk_t block, unsigned long count)
|
|
|
+{
|
|
|
+ struct buffer_head *bitmap_bh = NULL;
|
|
|
+ struct buffer_head *gd_bh;
|
|
|
+ ext4_group_t block_group;
|
|
|
+ ext4_grpblk_t bit;
|
|
|
+ unsigned int i;
|
|
|
+ struct ext4_group_desc *desc;
|
|
|
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
+ struct ext4_buddy e4b;
|
|
|
+ int err = 0, ret, blk_free_count;
|
|
|
+ ext4_grpblk_t blocks_freed;
|
|
|
+ struct ext4_group_info *grp;
|
|
|
+
|
|
|
+ ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);
|
|
|
+
|
|
|
+ ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
|
|
|
+ grp = ext4_get_group_info(sb, block_group);
|
|
|
+ /*
|
|
|
+ * Check to see if we are freeing blocks across a group
|
|
|
+ * boundary.
|
|
|
+ */
|
|
|
+ if (bit + count > EXT4_BLOCKS_PER_GROUP(sb))
|
|
|
+ goto error_return;
|
|
|
+
|
|
|
+ bitmap_bh = ext4_read_block_bitmap(sb, block_group);
|
|
|
+ if (!bitmap_bh)
|
|
|
+ goto error_return;
|
|
|
+ desc = ext4_get_group_desc(sb, block_group, &gd_bh);
|
|
|
+ if (!desc)
|
|
|
+ goto error_return;
|
|
|
+
|
|
|
+ if (in_range(ext4_block_bitmap(sb, desc), block, count) ||
|
|
|
+ in_range(ext4_inode_bitmap(sb, desc), block, count) ||
|
|
|
+ in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) ||
|
|
|
+ in_range(block + count - 1, ext4_inode_table(sb, desc),
|
|
|
+ sbi->s_itb_per_group)) {
|
|
|
+ ext4_error(sb, "Adding blocks in system zones - "
|
|
|
+ "Block = %llu, count = %lu",
|
|
|
+ block, count);
|
|
|
+ goto error_return;
|
|
|
+ }
|
|
|
+
|
|
|
+ BUFFER_TRACE(bitmap_bh, "getting write access");
|
|
|
+ err = ext4_journal_get_write_access(handle, bitmap_bh);
|
|
|
+ if (err)
|
|
|
+ goto error_return;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * We are about to modify some metadata. Call the journal APIs
|
|
|
+ * to unshare ->b_data if a currently-committing transaction is
|
|
|
+ * using it
|
|
|
+ */
|
|
|
+ BUFFER_TRACE(gd_bh, "get_write_access");
|
|
|
+ err = ext4_journal_get_write_access(handle, gd_bh);
|
|
|
+ if (err)
|
|
|
+ goto error_return;
|
|
|
+
|
|
|
+ for (i = 0, blocks_freed = 0; i < count; i++) {
|
|
|
+ BUFFER_TRACE(bitmap_bh, "clear bit");
|
|
|
+ if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {
|
|
|
+ ext4_error(sb, "bit already cleared for block %llu",
|
|
|
+ (ext4_fsblk_t)(block + i));
|
|
|
+ BUFFER_TRACE(bitmap_bh, "bit already cleared");
|
|
|
+ } else {
|
|
|
+ blocks_freed++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ err = ext4_mb_load_buddy(sb, block_group, &e4b);
|
|
|
+ if (err)
|
|
|
+ goto error_return;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * need to update group_info->bb_free and bitmap
|
|
|
+ * with group lock held. generate_buddy look at
|
|
|
+ * them with group lock_held
|
|
|
+ */
|
|
|
+ ext4_lock_group(sb, block_group);
|
|
|
+ mb_clear_bits(bitmap_bh->b_data, bit, count);
|
|
|
+ mb_free_blocks(NULL, &e4b, bit, count);
|
|
|
+ blk_free_count = blocks_freed + ext4_free_blks_count(sb, desc);
|
|
|
+ ext4_free_blks_set(sb, desc, blk_free_count);
|
|
|
+ desc->bg_checksum = ext4_group_desc_csum(sbi, block_group, desc);
|
|
|
+ ext4_unlock_group(sb, block_group);
|
|
|
+ percpu_counter_add(&sbi->s_freeblocks_counter, blocks_freed);
|
|
|
+
|
|
|
+ if (sbi->s_log_groups_per_flex) {
|
|
|
+ ext4_group_t flex_group = ext4_flex_group(sbi, block_group);
|
|
|
+ atomic_add(blocks_freed,
|
|
|
+ &sbi->s_flex_groups[flex_group].free_blocks);
|
|
|
+ }
|
|
|
+
|
|
|
+ ext4_mb_unload_buddy(&e4b);
|
|
|
+
|
|
|
+ /* We dirtied the bitmap block */
|
|
|
+ BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
|
|
|
+ err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
|
|
|
+
|
|
|
+ /* And the group descriptor block */
|
|
|
+ BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
|
|
|
+ ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh);
|
|
|
+ if (!err)
|
|
|
+ err = ret;
|
|
|
+
|
|
|
+error_return:
|
|
|
+ brelse(bitmap_bh);
|
|
|
+ ext4_std_error(sb, err);
|
|
|
+ return;
|
|
|
+}
|
|
|
+
|
|
|
/**
|
|
|
* ext4_trim_extent -- function to TRIM one single free extent in the group
|
|
|
* @sb: super block for the file system
|
|
@@ -4715,11 +4777,10 @@ error_return:
|
|
|
* one will allocate those blocks, mark it as used in buddy bitmap. This must
|
|
|
* be called with under the group lock.
|
|
|
*/
|
|
|
-static int ext4_trim_extent(struct super_block *sb, int start, int count,
|
|
|
- ext4_group_t group, struct ext4_buddy *e4b)
|
|
|
+static void ext4_trim_extent(struct super_block *sb, int start, int count,
|
|
|
+ ext4_group_t group, struct ext4_buddy *e4b)
|
|
|
{
|
|
|
struct ext4_free_extent ex;
|
|
|
- int ret = 0;
|
|
|
|
|
|
assert_spin_locked(ext4_group_lock_ptr(sb, group));
|
|
|
|
|
@@ -4733,12 +4794,9 @@ static int ext4_trim_extent(struct super_block *sb, int start, int count,
|
|
|
*/
|
|
|
mb_mark_used(e4b, &ex);
|
|
|
ext4_unlock_group(sb, group);
|
|
|
-
|
|
|
- ret = ext4_issue_discard(sb, group, start, count);
|
|
|
-
|
|
|
+ ext4_issue_discard(sb, group, start, count);
|
|
|
ext4_lock_group(sb, group);
|
|
|
mb_free_blocks(NULL, e4b, start, ex.fe_len);
|
|
|
- return ret;
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -4760,21 +4818,26 @@ static int ext4_trim_extent(struct super_block *sb, int start, int count,
|
|
|
* the group buddy bitmap. This is done until whole group is scanned.
|
|
|
*/
|
|
|
static ext4_grpblk_t
|
|
|
-ext4_trim_all_free(struct super_block *sb, struct ext4_buddy *e4b,
|
|
|
- ext4_grpblk_t start, ext4_grpblk_t max, ext4_grpblk_t minblocks)
|
|
|
+ext4_trim_all_free(struct super_block *sb, ext4_group_t group,
|
|
|
+ ext4_grpblk_t start, ext4_grpblk_t max,
|
|
|
+ ext4_grpblk_t minblocks)
|
|
|
{
|
|
|
void *bitmap;
|
|
|
ext4_grpblk_t next, count = 0;
|
|
|
- ext4_group_t group;
|
|
|
- int ret = 0;
|
|
|
+ struct ext4_buddy e4b;
|
|
|
+ int ret;
|
|
|
|
|
|
- BUG_ON(e4b == NULL);
|
|
|
+ ret = ext4_mb_load_buddy(sb, group, &e4b);
|
|
|
+ if (ret) {
|
|
|
+ ext4_error(sb, "Error in loading buddy "
|
|
|
+ "information for %u", group);
|
|
|
+ return ret;
|
|
|
+ }
|
|
|
+ bitmap = e4b.bd_bitmap;
|
|
|
|
|
|
- bitmap = e4b->bd_bitmap;
|
|
|
- group = e4b->bd_group;
|
|
|
- start = (e4b->bd_info->bb_first_free > start) ?
|
|
|
- e4b->bd_info->bb_first_free : start;
|
|
|
ext4_lock_group(sb, group);
|
|
|
+ start = (e4b.bd_info->bb_first_free > start) ?
|
|
|
+ e4b.bd_info->bb_first_free : start;
|
|
|
|
|
|
while (start < max) {
|
|
|
start = mb_find_next_zero_bit(bitmap, max, start);
|
|
@@ -4783,10 +4846,8 @@ ext4_trim_all_free(struct super_block *sb, struct ext4_buddy *e4b,
|
|
|
next = mb_find_next_bit(bitmap, max, start);
|
|
|
|
|
|
if ((next - start) >= minblocks) {
|
|
|
- ret = ext4_trim_extent(sb, start,
|
|
|
- next - start, group, e4b);
|
|
|
- if (ret < 0)
|
|
|
- break;
|
|
|
+ ext4_trim_extent(sb, start,
|
|
|
+ next - start, group, &e4b);
|
|
|
count += next - start;
|
|
|
}
|
|
|
start = next + 1;
|
|
@@ -4802,17 +4863,15 @@ ext4_trim_all_free(struct super_block *sb, struct ext4_buddy *e4b,
|
|
|
ext4_lock_group(sb, group);
|
|
|
}
|
|
|
|
|
|
- if ((e4b->bd_info->bb_free - count) < minblocks)
|
|
|
+ if ((e4b.bd_info->bb_free - count) < minblocks)
|
|
|
break;
|
|
|
}
|
|
|
ext4_unlock_group(sb, group);
|
|
|
+ ext4_mb_unload_buddy(&e4b);
|
|
|
|
|
|
ext4_debug("trimmed %d blocks in the group %d\n",
|
|
|
count, group);
|
|
|
|
|
|
- if (ret < 0)
|
|
|
- count = ret;
|
|
|
-
|
|
|
return count;
|
|
|
}
|
|
|
|
|
@@ -4830,11 +4889,11 @@ ext4_trim_all_free(struct super_block *sb, struct ext4_buddy *e4b,
|
|
|
*/
|
|
|
int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
|
|
|
{
|
|
|
- struct ext4_buddy e4b;
|
|
|
+ struct ext4_group_info *grp;
|
|
|
ext4_group_t first_group, last_group;
|
|
|
ext4_group_t group, ngroups = ext4_get_groups_count(sb);
|
|
|
ext4_grpblk_t cnt = 0, first_block, last_block;
|
|
|
- uint64_t start, len, minlen, trimmed;
|
|
|
+ uint64_t start, len, minlen, trimmed = 0;
|
|
|
ext4_fsblk_t first_data_blk =
|
|
|
le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
|
int ret = 0;
|
|
@@ -4842,7 +4901,6 @@ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
|
|
|
start = range->start >> sb->s_blocksize_bits;
|
|
|
len = range->len >> sb->s_blocksize_bits;
|
|
|
minlen = range->minlen >> sb->s_blocksize_bits;
|
|
|
- trimmed = 0;
|
|
|
|
|
|
if (unlikely(minlen > EXT4_BLOCKS_PER_GROUP(sb)))
|
|
|
return -EINVAL;
|
|
@@ -4863,11 +4921,12 @@ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
|
|
|
return -EINVAL;
|
|
|
|
|
|
for (group = first_group; group <= last_group; group++) {
|
|
|
- ret = ext4_mb_load_buddy(sb, group, &e4b);
|
|
|
- if (ret) {
|
|
|
- ext4_error(sb, "Error in loading buddy "
|
|
|
- "information for %u", group);
|
|
|
- break;
|
|
|
+ grp = ext4_get_group_info(sb, group);
|
|
|
+ /* We only do this if the grp has never been initialized */
|
|
|
+ if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
|
|
|
+ ret = ext4_mb_init_group(sb, group);
|
|
|
+ if (ret)
|
|
|
+ break;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -4880,16 +4939,14 @@ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
|
|
|
last_block = first_block + len;
|
|
|
len -= last_block - first_block;
|
|
|
|
|
|
- if (e4b.bd_info->bb_free >= minlen) {
|
|
|
- cnt = ext4_trim_all_free(sb, &e4b, first_block,
|
|
|
+ if (grp->bb_free >= minlen) {
|
|
|
+ cnt = ext4_trim_all_free(sb, group, first_block,
|
|
|
last_block, minlen);
|
|
|
if (cnt < 0) {
|
|
|
ret = cnt;
|
|
|
- ext4_mb_unload_buddy(&e4b);
|
|
|
break;
|
|
|
}
|
|
|
}
|
|
|
- ext4_mb_unload_buddy(&e4b);
|
|
|
trimmed += cnt;
|
|
|
first_block = 0;
|
|
|
}
|