|
@@ -37,11 +37,19 @@
|
|
|
|
|
|
/*
|
|
|
* Usage:
|
|
|
- * dcache_hash_lock protects dcache hash table, s_anon lists
|
|
|
+ * dcache_hash_lock protects:
|
|
|
+ * - the dcache hash table, s_anon lists
|
|
|
+ * dcache_lru_lock protects:
|
|
|
+ * - the dcache lru lists and counters
|
|
|
+ * d_lock protects:
|
|
|
+ * - d_flags
|
|
|
+ * - d_name
|
|
|
+ * - d_lru
|
|
|
*
|
|
|
* Ordering:
|
|
|
* dcache_lock
|
|
|
* dentry->d_lock
|
|
|
+ * dcache_lru_lock
|
|
|
* dcache_hash_lock
|
|
|
*
|
|
|
* if (dentry1 < dentry2)
|
|
@@ -52,6 +60,7 @@ int sysctl_vfs_cache_pressure __read_mostly = 100;
|
|
|
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
|
|
|
|
|
|
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
|
|
|
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
|
|
|
__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
|
|
|
__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
|
|
|
|
|
@@ -154,28 +163,38 @@ static void dentry_iput(struct dentry * dentry)
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * dentry_lru_(add|del|move_tail) must be called with dcache_lock held.
|
|
|
+ * dentry_lru_(add|del|move_tail) must be called with d_lock held.
|
|
|
*/
|
|
|
static void dentry_lru_add(struct dentry *dentry)
|
|
|
{
|
|
|
if (list_empty(&dentry->d_lru)) {
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
|
|
|
dentry->d_sb->s_nr_dentry_unused++;
|
|
|
dentry_stat.nr_unused++;
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+static void __dentry_lru_del(struct dentry *dentry)
|
|
|
+{
|
|
|
+ list_del_init(&dentry->d_lru);
|
|
|
+ dentry->d_sb->s_nr_dentry_unused--;
|
|
|
+ dentry_stat.nr_unused--;
|
|
|
+}
|
|
|
+
|
|
|
static void dentry_lru_del(struct dentry *dentry)
|
|
|
{
|
|
|
if (!list_empty(&dentry->d_lru)) {
|
|
|
- list_del_init(&dentry->d_lru);
|
|
|
- dentry->d_sb->s_nr_dentry_unused--;
|
|
|
- dentry_stat.nr_unused--;
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
+ __dentry_lru_del(dentry);
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static void dentry_lru_move_tail(struct dentry *dentry)
|
|
|
{
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
if (list_empty(&dentry->d_lru)) {
|
|
|
list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
|
|
|
dentry->d_sb->s_nr_dentry_unused++;
|
|
@@ -183,6 +202,7 @@ static void dentry_lru_move_tail(struct dentry *dentry)
|
|
|
} else {
|
|
|
list_move_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
|
|
|
}
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -192,6 +212,8 @@ static void dentry_lru_move_tail(struct dentry *dentry)
|
|
|
* The dentry must already be unhashed and removed from the LRU.
|
|
|
*
|
|
|
* If this is the root of the dentry tree, return NULL.
|
|
|
+ *
|
|
|
+ * dcache_lock and d_lock must be held by caller, are dropped by d_kill.
|
|
|
*/
|
|
|
static struct dentry *d_kill(struct dentry *dentry)
|
|
|
__releases(dentry->d_lock)
|
|
@@ -383,10 +405,19 @@ int d_invalidate(struct dentry * dentry)
|
|
|
EXPORT_SYMBOL(d_invalidate);
|
|
|
|
|
|
/* This should be called _only_ with dcache_lock held */
|
|
|
+static inline struct dentry * __dget_locked_dlock(struct dentry *dentry)
|
|
|
+{
|
|
|
+ atomic_inc(&dentry->d_count);
|
|
|
+ dentry_lru_del(dentry);
|
|
|
+ return dentry;
|
|
|
+}
|
|
|
+
|
|
|
static inline struct dentry * __dget_locked(struct dentry *dentry)
|
|
|
{
|
|
|
atomic_inc(&dentry->d_count);
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
dentry_lru_del(dentry);
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
return dentry;
|
|
|
}
|
|
|
|
|
@@ -465,7 +496,7 @@ restart:
|
|
|
list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
|
|
|
spin_lock(&dentry->d_lock);
|
|
|
if (!atomic_read(&dentry->d_count)) {
|
|
|
- __dget_locked(dentry);
|
|
|
+ __dget_locked_dlock(dentry);
|
|
|
__d_drop(dentry);
|
|
|
spin_unlock(&dentry->d_lock);
|
|
|
spin_unlock(&dcache_lock);
|
|
@@ -489,7 +520,6 @@ EXPORT_SYMBOL(d_prune_aliases);
|
|
|
static void prune_one_dentry(struct dentry * dentry)
|
|
|
__releases(dentry->d_lock)
|
|
|
__releases(dcache_lock)
|
|
|
- __acquires(dcache_lock)
|
|
|
{
|
|
|
__d_drop(dentry);
|
|
|
dentry = d_kill(dentry);
|
|
@@ -498,15 +528,16 @@ static void prune_one_dentry(struct dentry * dentry)
|
|
|
* Prune ancestors. Locking is simpler than in dput(),
|
|
|
* because dcache_lock needs to be taken anyway.
|
|
|
*/
|
|
|
- spin_lock(&dcache_lock);
|
|
|
while (dentry) {
|
|
|
- if (!atomic_dec_and_lock(&dentry->d_count, &dentry->d_lock))
|
|
|
+ spin_lock(&dcache_lock);
|
|
|
+ if (!atomic_dec_and_lock(&dentry->d_count, &dentry->d_lock)) {
|
|
|
+ spin_unlock(&dcache_lock);
|
|
|
return;
|
|
|
+ }
|
|
|
|
|
|
dentry_lru_del(dentry);
|
|
|
__d_drop(dentry);
|
|
|
dentry = d_kill(dentry);
|
|
|
- spin_lock(&dcache_lock);
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -516,21 +547,31 @@ static void shrink_dentry_list(struct list_head *list)
|
|
|
|
|
|
while (!list_empty(list)) {
|
|
|
dentry = list_entry(list->prev, struct dentry, d_lru);
|
|
|
- dentry_lru_del(dentry);
|
|
|
+
|
|
|
+ if (!spin_trylock(&dentry->d_lock)) {
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
+ cpu_relax();
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ __dentry_lru_del(dentry);
|
|
|
|
|
|
/*
|
|
|
* We found an inuse dentry which was not removed from
|
|
|
* the LRU because of laziness during lookup. Do not free
|
|
|
* it - just keep it off the LRU list.
|
|
|
*/
|
|
|
- spin_lock(&dentry->d_lock);
|
|
|
if (atomic_read(&dentry->d_count)) {
|
|
|
spin_unlock(&dentry->d_lock);
|
|
|
continue;
|
|
|
}
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
+
|
|
|
prune_one_dentry(dentry);
|
|
|
- /* dentry->d_lock was dropped in prune_one_dentry() */
|
|
|
- cond_resched_lock(&dcache_lock);
|
|
|
+ /* dcache_lock and dentry->d_lock dropped */
|
|
|
+ spin_lock(&dcache_lock);
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -551,32 +592,36 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
|
|
|
int cnt = *count;
|
|
|
|
|
|
spin_lock(&dcache_lock);
|
|
|
+relock:
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
while (!list_empty(&sb->s_dentry_lru)) {
|
|
|
dentry = list_entry(sb->s_dentry_lru.prev,
|
|
|
struct dentry, d_lru);
|
|
|
BUG_ON(dentry->d_sb != sb);
|
|
|
|
|
|
+ if (!spin_trylock(&dentry->d_lock)) {
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
+ cpu_relax();
|
|
|
+ goto relock;
|
|
|
+ }
|
|
|
+
|
|
|
/*
|
|
|
* If we are honouring the DCACHE_REFERENCED flag and the
|
|
|
* dentry has this flag set, don't free it. Clear the flag
|
|
|
* and put it back on the LRU.
|
|
|
*/
|
|
|
- if (flags & DCACHE_REFERENCED) {
|
|
|
- spin_lock(&dentry->d_lock);
|
|
|
- if (dentry->d_flags & DCACHE_REFERENCED) {
|
|
|
- dentry->d_flags &= ~DCACHE_REFERENCED;
|
|
|
- list_move(&dentry->d_lru, &referenced);
|
|
|
- spin_unlock(&dentry->d_lock);
|
|
|
- cond_resched_lock(&dcache_lock);
|
|
|
- continue;
|
|
|
- }
|
|
|
+ if (flags & DCACHE_REFERENCED &&
|
|
|
+ dentry->d_flags & DCACHE_REFERENCED) {
|
|
|
+ dentry->d_flags &= ~DCACHE_REFERENCED;
|
|
|
+ list_move(&dentry->d_lru, &referenced);
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+ } else {
|
|
|
+ list_move_tail(&dentry->d_lru, &tmp);
|
|
|
spin_unlock(&dentry->d_lock);
|
|
|
+ if (!--cnt)
|
|
|
+ break;
|
|
|
}
|
|
|
-
|
|
|
- list_move_tail(&dentry->d_lru, &tmp);
|
|
|
- if (!--cnt)
|
|
|
- break;
|
|
|
- cond_resched_lock(&dcache_lock);
|
|
|
+ /* XXX: re-add cond_resched_lock when dcache_lock goes away */
|
|
|
}
|
|
|
|
|
|
*count = cnt;
|
|
@@ -584,6 +629,7 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
|
|
|
|
|
|
if (!list_empty(&referenced))
|
|
|
list_splice(&referenced, &sb->s_dentry_lru);
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
spin_unlock(&dcache_lock);
|
|
|
|
|
|
}
|
|
@@ -679,10 +725,12 @@ void shrink_dcache_sb(struct super_block *sb)
|
|
|
LIST_HEAD(tmp);
|
|
|
|
|
|
spin_lock(&dcache_lock);
|
|
|
+ spin_lock(&dcache_lru_lock);
|
|
|
while (!list_empty(&sb->s_dentry_lru)) {
|
|
|
list_splice_init(&sb->s_dentry_lru, &tmp);
|
|
|
shrink_dentry_list(&tmp);
|
|
|
}
|
|
|
+ spin_unlock(&dcache_lru_lock);
|
|
|
spin_unlock(&dcache_lock);
|
|
|
}
|
|
|
EXPORT_SYMBOL(shrink_dcache_sb);
|
|
@@ -701,7 +749,9 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
|
|
|
|
|
|
/* detach this root from the system */
|
|
|
spin_lock(&dcache_lock);
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
dentry_lru_del(dentry);
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
__d_drop(dentry);
|
|
|
spin_unlock(&dcache_lock);
|
|
|
|
|
@@ -715,7 +765,9 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
|
|
|
spin_lock(&dcache_lock);
|
|
|
list_for_each_entry(loop, &dentry->d_subdirs,
|
|
|
d_u.d_child) {
|
|
|
+ spin_lock(&loop->d_lock);
|
|
|
dentry_lru_del(loop);
|
|
|
+ spin_unlock(&loop->d_lock);
|
|
|
__d_drop(loop);
|
|
|
cond_resched_lock(&dcache_lock);
|
|
|
}
|
|
@@ -892,6 +944,8 @@ resume:
|
|
|
struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
|
|
|
next = tmp->next;
|
|
|
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
+
|
|
|
/*
|
|
|
* move only zero ref count dentries to the end
|
|
|
* of the unused list for prune_dcache
|
|
@@ -903,6 +957,8 @@ resume:
|
|
|
dentry_lru_del(dentry);
|
|
|
}
|
|
|
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+
|
|
|
/*
|
|
|
* We can return to the caller if we have found some (this
|
|
|
* ensures forward progress). We'll be coming back to find
|