Btrfs: Add shared reference cache

Btrfs has a cache of reference counts in leaves, allowing it to
avoid reading tree leaves while deleting snapshots.  To reduce
contention with multiple subvolumes, this cache is private to each
subvolume.

This patch adds shared reference cache support. The new space
balancing code plays with multiple subvols at the same time, So
the old per-subvol reference cache is not well suited.

Signed-off-by: Chris Mason <chris.mason@oracle.com>

fs/btrfs/ctree.h

@@ -81,6 +81,10 @@ struct btrfs_ordered_sum;
 #define BTRFS_TREE_LOG_OBJECTID -6ULL
 #define BTRFS_TREE_LOG_FIXUP_OBJECTID -7ULL
 
+/* for space balancing */
+#define BTRFS_TREE_RELOC_OBJECTID -8ULL
+#define BTRFS_DATA_RELOC_TREE_OBJECTID -9ULL
+
 /* dummy objectid represents multiple objectids */
 #define BTRFS_MULTIPLE_OBJECTIDS -255ULL
 
@@ -539,6 +543,12 @@ struct btrfs_block_group_cache {
 	struct list_head list;
 };
 
+struct btrfs_leaf_ref_tree {
+	struct rb_root root;
+	struct list_head list;
+	spinlock_t lock;
+};
+
 struct btrfs_device;
 struct btrfs_fs_devices;
 struct btrfs_fs_info {
@@ -637,6 +647,8 @@ struct btrfs_fs_info {
 	struct task_struct *cleaner_kthread;
 	int thread_pool_size;
 
+	struct btrfs_leaf_ref_tree shared_ref_tree;
+
 	struct kobject super_kobj;
 	struct completion kobj_unregister;
 	int do_barriers;
@@ -670,13 +682,6 @@ struct btrfs_fs_info {
 	void *bdev_holder;
 };
 
-struct btrfs_leaf_ref_tree {
-	struct rb_root root;
-	struct btrfs_leaf_ref *last;
-	struct list_head list;
-	spinlock_t lock;
-};
-
 /*
  * in ram representation of the tree.  extent_root is used for all allocations
  * and for the extent tree extent_root root.

fs/btrfs/disk-io.c

@@ -1406,6 +1406,8 @@ struct btrfs_root *open_ctree(struct super_block *sb,
 			     fs_info->btree_inode->i_mapping, GFP_NOFS);
 	fs_info->do_barriers = 1;
 
+	btrfs_leaf_ref_tree_init(&fs_info->shared_ref_tree);
+
 	BTRFS_I(fs_info->btree_inode)->root = tree_root;
 	memset(&BTRFS_I(fs_info->btree_inode)->location, 0,
 	       sizeof(struct btrfs_key));

fs/btrfs/extent-tree.c

@@ -1091,16 +1091,26 @@ out:
 int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		    struct extent_buffer *buf, u32 nr_extents)
 {
-	u32 nritems;
 	struct btrfs_key key;
 	struct btrfs_file_extent_item *fi;
+	u64 root_gen;
+	u32 nritems;
 	int i;
 	int level;
 	int ret = 0;
+	int shared = 0;
 
 	if (!root->ref_cows)
 		return 0;
 
+	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
+		shared = 0;
+		root_gen = root->root_key.offset;
+	} else {
+		shared = 1;
+		root_gen = trans->transid - 1;
+	}
+
 	level = btrfs_header_level(buf);
 	nritems = btrfs_header_nritems(buf);
 
@@ -1114,7 +1124,7 @@ int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 			goto out;
 		}
 
-		ref->root_gen = root->root_key.offset;
+		ref->root_gen = root_gen;
 		ref->bytenr = buf->start;
 		ref->owner = btrfs_header_owner(buf);
 		ref->generation = btrfs_header_generation(buf);
@@ -1143,8 +1153,7 @@ int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 			info++;
 		}
 
-		BUG_ON(!root->ref_tree);
-		ret = btrfs_add_leaf_ref(root, ref);
+		ret = btrfs_add_leaf_ref(root, ref, shared);
 		WARN_ON(ret);
 		btrfs_free_leaf_ref(root, ref);
 	}

fs/btrfs/ref-cache.c

@@ -78,7 +78,6 @@ static struct rb_node *tree_insert(struct rb_root *root, u64 bytenr,
 	}
 
 	entry = rb_entry(node, struct btrfs_leaf_ref, rb_node);
-	entry->in_tree = 1;
 	rb_link_node(node, parent, p);
 	rb_insert_color(node, root);
 	return NULL;
@@ -103,23 +102,29 @@ static struct rb_node *tree_search(struct rb_root *root, u64 bytenr)
 	return NULL;
 }
 
-int btrfs_remove_leaf_refs(struct btrfs_root *root, u64 max_root_gen)
+int btrfs_remove_leaf_refs(struct btrfs_root *root, u64 max_root_gen,
+			   int shared)
 {
 	struct btrfs_leaf_ref *ref = NULL;
 	struct btrfs_leaf_ref_tree *tree = root->ref_tree;
 
+	if (shared)
+		tree = &root->fs_info->shared_ref_tree;
 	if (!tree)
 		return 0;
 
 	spin_lock(&tree->lock);
 	while(!list_empty(&tree->list)) {
 		ref = list_entry(tree->list.next, struct btrfs_leaf_ref, list);
-		BUG_ON(!ref->in_tree);
+		BUG_ON(ref->tree != tree);
 		if (ref->root_gen > max_root_gen)
 			break;
+		if (!xchg(&ref->in_tree, 0)) {
+			cond_resched_lock(&tree->lock);
+			continue;
+		}
 
 		rb_erase(&ref->rb_node, &tree->root);
-		ref->in_tree = 0;
 		list_del_init(&ref->list);
 
 		spin_unlock(&tree->lock);
@@ -137,32 +142,43 @@ struct btrfs_leaf_ref *btrfs_lookup_leaf_ref(struct btrfs_root *root,
 	struct rb_node *rb;
 	struct btrfs_leaf_ref *ref = NULL;
 	struct btrfs_leaf_ref_tree *tree = root->ref_tree;
-
-	if (!tree)
-		return NULL;
-
-	spin_lock(&tree->lock);
-	rb = tree_search(&tree->root, bytenr);
-	if (rb)
-		ref = rb_entry(rb, struct btrfs_leaf_ref, rb_node);
-	if (ref)
-		atomic_inc(&ref->usage);
-	spin_unlock(&tree->lock);
-	return ref;
+again:
+	if (tree) {
+		spin_lock(&tree->lock);
+		rb = tree_search(&tree->root, bytenr);
+		if (rb)
+			ref = rb_entry(rb, struct btrfs_leaf_ref, rb_node);
+		if (ref)
+			atomic_inc(&ref->usage);
+		spin_unlock(&tree->lock);
+		if (ref)
+			return ref;
+	}
+	if (tree != &root->fs_info->shared_ref_tree) {
+		tree = &root->fs_info->shared_ref_tree;
+		goto again;
+	}
+	return NULL;
 }
 
-int btrfs_add_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref)
+int btrfs_add_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref,
+		       int shared)
 {
 	int ret = 0;
 	struct rb_node *rb;
 	struct btrfs_leaf_ref_tree *tree = root->ref_tree;
 
+	if (shared)
+		tree = &root->fs_info->shared_ref_tree;
+
 	spin_lock(&tree->lock);
 	rb = tree_insert(&tree->root, ref->bytenr, &ref->rb_node);
 	if (rb) {
 		ret = -EEXIST;
 	} else {
 		atomic_inc(&ref->usage);
+		ref->tree = tree;
+		ref->in_tree = 1;
 		list_add_tail(&ref->list, &tree->list);
 	}
 	spin_unlock(&tree->lock);
@@ -171,13 +187,15 @@ int btrfs_add_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref)
 
 int btrfs_remove_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref)
 {
-	struct btrfs_leaf_ref_tree *tree = root->ref_tree;
+	struct btrfs_leaf_ref_tree *tree;
+
+	if (!xchg(&ref->in_tree, 0))
+		return 0;
 
-	BUG_ON(!ref->in_tree);
+	tree = ref->tree;
 	spin_lock(&tree->lock);
 
 	rb_erase(&ref->rb_node, &tree->root);
-	ref->in_tree = 0;
 	list_del_init(&ref->list);
 
 	spin_unlock(&tree->lock);

fs/btrfs/ref-cache.h

@@ -27,6 +27,7 @@ struct btrfs_extent_info {
 
 struct btrfs_leaf_ref {
 	struct rb_node rb_node;
+	struct btrfs_leaf_ref_tree *tree;
 	int in_tree;
 	atomic_t usage;
 
@@ -64,8 +65,10 @@ struct btrfs_leaf_ref *btrfs_alloc_leaf_ref(struct btrfs_root *root,
 void btrfs_free_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref);
 struct btrfs_leaf_ref *btrfs_lookup_leaf_ref(struct btrfs_root *root,
 					     u64 bytenr);
-int btrfs_add_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref);
-int btrfs_remove_leaf_refs(struct btrfs_root *root, u64 max_root_gen);
+int btrfs_add_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref,
+		       int shared);
+int btrfs_remove_leaf_refs(struct btrfs_root *root, u64 max_root_gen,
+			   int shared);
 int btrfs_remove_leaf_ref(struct btrfs_root *root, struct btrfs_leaf_ref *ref);
 
 #endif

fs/btrfs/transaction.c

@@ -650,7 +650,7 @@ static noinline int drop_dirty_roots(struct btrfs_root *tree_root,
 		ret = btrfs_end_transaction(trans, tree_root);
 		BUG_ON(ret);
 
-		ret = btrfs_remove_leaf_refs(root, max_useless);
+		ret = btrfs_remove_leaf_refs(root, max_useless, 0);
 		BUG_ON(ret);
 
 		free_extent_buffer(dirty->root->node);