|
@@ -145,7 +145,11 @@ static inline void inc_zcache_obj_count(void)
|
|
|
if (zcache_obj_count > zcache_obj_count_max)
|
|
|
zcache_obj_count_max = zcache_obj_count;
|
|
|
}
|
|
|
-
|
|
|
+static inline void dec_zcache_obj_count(void)
|
|
|
+{
|
|
|
+ zcache_obj_count = atomic_dec_return(&zcache_obj_atomic);
|
|
|
+ BUG_ON(zcache_obj_count < 0);
|
|
|
+};
|
|
|
static atomic_t zcache_objnode_atomic = ATOMIC_INIT(0);
|
|
|
static ssize_t zcache_objnode_count_max;
|
|
|
static inline void inc_zcache_objnode_count(void)
|
|
@@ -154,6 +158,11 @@ static inline void inc_zcache_objnode_count(void)
|
|
|
if (zcache_objnode_count > zcache_objnode_count_max)
|
|
|
zcache_objnode_count_max = zcache_objnode_count;
|
|
|
};
|
|
|
+static inline void dec_zcache_objnode_count(void)
|
|
|
+{
|
|
|
+ zcache_objnode_count = atomic_dec_return(&zcache_objnode_atomic);
|
|
|
+ BUG_ON(zcache_objnode_count < 0);
|
|
|
+};
|
|
|
static u64 zcache_eph_zbytes;
|
|
|
static atomic_long_t zcache_eph_zbytes_atomic = ATOMIC_INIT(0);
|
|
|
static u64 zcache_eph_zbytes_max;
|
|
@@ -163,6 +172,10 @@ static inline void inc_zcache_eph_zbytes(unsigned clen)
|
|
|
if (zcache_eph_zbytes > zcache_eph_zbytes_max)
|
|
|
zcache_eph_zbytes_max = zcache_eph_zbytes;
|
|
|
};
|
|
|
+static inline void dec_zcache_eph_zbytes(unsigned zsize)
|
|
|
+{
|
|
|
+ zcache_eph_zbytes = atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
|
|
|
+};
|
|
|
static u64 zcache_pers_zbytes;
|
|
|
static atomic_long_t zcache_pers_zbytes_atomic = ATOMIC_INIT(0);
|
|
|
static u64 zcache_pers_zbytes_max;
|
|
@@ -173,6 +186,10 @@ static inline void inc_zcache_pers_zbytes(unsigned clen)
|
|
|
if (zcache_pers_zbytes > zcache_pers_zbytes_max)
|
|
|
zcache_pers_zbytes_max = zcache_pers_zbytes;
|
|
|
}
|
|
|
+static inline void dec_zcache_pers_zbytes(unsigned zsize)
|
|
|
+{
|
|
|
+ zcache_pers_zbytes = atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
|
|
|
+}
|
|
|
static atomic_t zcache_eph_pageframes_atomic = ATOMIC_INIT(0);
|
|
|
static ssize_t zcache_eph_pageframes_max;
|
|
|
static ssize_t zcache_pers_pageframes;
|
|
@@ -182,6 +199,10 @@ static inline void inc_zcache_eph_pageframes(void)
|
|
|
if (zcache_eph_pageframes > zcache_eph_pageframes_max)
|
|
|
zcache_eph_pageframes_max = zcache_eph_pageframes;
|
|
|
};
|
|
|
+static inline void dec_zcache_eph_pageframes(void)
|
|
|
+{
|
|
|
+ zcache_eph_pageframes = atomic_dec_return(&zcache_eph_pageframes_atomic);
|
|
|
+};
|
|
|
static atomic_t zcache_pers_pageframes_atomic = ATOMIC_INIT(0);
|
|
|
static ssize_t zcache_pers_pageframes_max;
|
|
|
static ssize_t zcache_pageframes_alloced;
|
|
@@ -191,6 +212,10 @@ static inline void inc_zcache_pers_pageframes(void)
|
|
|
if (zcache_pers_pageframes > zcache_pers_pageframes_max)
|
|
|
zcache_pers_pageframes_max = zcache_pers_pageframes;
|
|
|
}
|
|
|
+static inline void dec_zcache_pers_pageframes(void)
|
|
|
+{
|
|
|
+ zcache_pers_pageframes = atomic_dec_return(&zcache_pers_pageframes_atomic);
|
|
|
+}
|
|
|
static atomic_t zcache_pageframes_alloced_atomic = ATOMIC_INIT(0);
|
|
|
static ssize_t zcache_pageframes_freed;
|
|
|
static inline void inc_zcache_pageframes_alloced(void)
|
|
@@ -212,6 +237,10 @@ static inline void inc_zcache_eph_zpages(void)
|
|
|
if (zcache_eph_zpages > zcache_eph_zpages_max)
|
|
|
zcache_eph_zpages_max = zcache_eph_zpages;
|
|
|
}
|
|
|
+static inline void dec_zcache_eph_zpages(unsigned zpages)
|
|
|
+{
|
|
|
+ zcache_eph_zpages = atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
|
|
|
+}
|
|
|
static atomic_t zcache_pers_zpages_atomic = ATOMIC_INIT(0);
|
|
|
static ssize_t zcache_pers_zpages_max;
|
|
|
static inline void inc_zcache_pers_zpages(void)
|
|
@@ -220,6 +249,10 @@ static inline void inc_zcache_pers_zpages(void)
|
|
|
if (zcache_pers_zpages > zcache_pers_zpages_max)
|
|
|
zcache_pers_zpages_max = zcache_pers_zpages;
|
|
|
}
|
|
|
+static inline void dec_zcache_pers_zpages(unsigned zpages)
|
|
|
+{
|
|
|
+ zcache_pers_zpages = atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
|
|
|
+}
|
|
|
/* but for the rest of these, counting races are ok */
|
|
|
static ssize_t zcache_flush_total;
|
|
|
static ssize_t zcache_flush_found;
|
|
@@ -484,9 +517,7 @@ static struct tmem_objnode *zcache_objnode_alloc(struct tmem_pool *pool)
|
|
|
static void zcache_objnode_free(struct tmem_objnode *objnode,
|
|
|
struct tmem_pool *pool)
|
|
|
{
|
|
|
- zcache_objnode_count =
|
|
|
- atomic_dec_return(&zcache_objnode_atomic);
|
|
|
- BUG_ON(zcache_objnode_count < 0);
|
|
|
+ dec_zcache_objnode_count();
|
|
|
kmem_cache_free(zcache_objnode_cache, objnode);
|
|
|
}
|
|
|
|
|
@@ -505,9 +536,7 @@ static struct tmem_obj *zcache_obj_alloc(struct tmem_pool *pool)
|
|
|
|
|
|
static void zcache_obj_free(struct tmem_obj *obj, struct tmem_pool *pool)
|
|
|
{
|
|
|
- zcache_obj_count =
|
|
|
- atomic_dec_return(&zcache_obj_atomic);
|
|
|
- BUG_ON(zcache_obj_count < 0);
|
|
|
+ dec_zcache_obj_count();
|
|
|
kmem_cache_free(zcache_obj_cache, obj);
|
|
|
}
|
|
|
|
|
@@ -827,20 +856,14 @@ static int zcache_pampd_get_data_and_free(char *data, size_t *sizep, bool raw,
|
|
|
&zsize, &zpages);
|
|
|
if (eph) {
|
|
|
if (page)
|
|
|
- zcache_eph_pageframes =
|
|
|
- atomic_dec_return(&zcache_eph_pageframes_atomic);
|
|
|
- zcache_eph_zpages =
|
|
|
- atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
|
|
|
- zcache_eph_zbytes =
|
|
|
- atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
|
|
|
+ dec_zcache_eph_pageframes();
|
|
|
+ dec_zcache_eph_zpages(zpages);
|
|
|
+ dec_zcache_eph_zbytes(zsize);
|
|
|
} else {
|
|
|
if (page)
|
|
|
- zcache_pers_pageframes =
|
|
|
- atomic_dec_return(&zcache_pers_pageframes_atomic);
|
|
|
- zcache_pers_zpages =
|
|
|
- atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
|
|
|
- zcache_pers_zbytes =
|
|
|
- atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
|
|
|
+ dec_zcache_pers_pageframes();
|
|
|
+ dec_zcache_pers_zpages(zpages);
|
|
|
+ dec_zcache_pers_zbytes(zsize);
|
|
|
}
|
|
|
if (!is_local_client(pool->client))
|
|
|
ramster_count_foreign_pages(eph, -1);
|
|
@@ -870,23 +893,17 @@ static void zcache_pampd_free(void *pampd, struct tmem_pool *pool,
|
|
|
page = zbud_free_and_delist((struct zbudref *)pampd,
|
|
|
true, &zsize, &zpages);
|
|
|
if (page)
|
|
|
- zcache_eph_pageframes =
|
|
|
- atomic_dec_return(&zcache_eph_pageframes_atomic);
|
|
|
- zcache_eph_zpages =
|
|
|
- atomic_sub_return(zpages, &zcache_eph_zpages_atomic);
|
|
|
- zcache_eph_zbytes =
|
|
|
- atomic_long_sub_return(zsize, &zcache_eph_zbytes_atomic);
|
|
|
+ dec_zcache_eph_pageframes();
|
|
|
+ dec_zcache_eph_zpages(zpages);
|
|
|
+ dec_zcache_eph_zbytes(zsize);
|
|
|
/* FIXME CONFIG_RAMSTER... check acct parameter? */
|
|
|
} else {
|
|
|
page = zbud_free_and_delist((struct zbudref *)pampd,
|
|
|
false, &zsize, &zpages);
|
|
|
if (page)
|
|
|
- zcache_pers_pageframes =
|
|
|
- atomic_dec_return(&zcache_pers_pageframes_atomic);
|
|
|
- zcache_pers_zpages =
|
|
|
- atomic_sub_return(zpages, &zcache_pers_zpages_atomic);
|
|
|
- zcache_pers_zbytes =
|
|
|
- atomic_long_sub_return(zsize, &zcache_pers_zbytes_atomic);
|
|
|
+ dec_zcache_pers_pageframes();
|
|
|
+ dec_zcache_pers_zpages(zpages);
|
|
|
+ dec_zcache_pers_zbytes(zsize);
|
|
|
}
|
|
|
if (!is_local_client(pool->client))
|
|
|
ramster_count_foreign_pages(is_ephemeral(pool), -1);
|
|
@@ -1008,13 +1025,10 @@ static struct page *zcache_evict_eph_pageframe(void)
|
|
|
page = zbud_evict_pageframe_lru(&zsize, &zpages);
|
|
|
if (page == NULL)
|
|
|
goto out;
|
|
|
- zcache_eph_zbytes = atomic_long_sub_return(zsize,
|
|
|
- &zcache_eph_zbytes_atomic);
|
|
|
- zcache_eph_zpages = atomic_sub_return(zpages,
|
|
|
- &zcache_eph_zpages_atomic);
|
|
|
+ dec_zcache_eph_zbytes(zsize);
|
|
|
+ dec_zcache_eph_zpages(zpages);
|
|
|
zcache_evicted_eph_zpages += zpages;
|
|
|
- zcache_eph_pageframes =
|
|
|
- atomic_dec_return(&zcache_eph_pageframes_atomic);
|
|
|
+ dec_zcache_eph_pageframes();
|
|
|
zcache_evicted_eph_pageframes++;
|
|
|
out:
|
|
|
return page;
|
|
@@ -1029,6 +1043,11 @@ static inline void inc_zcache_outstanding_writeback_pages(void)
|
|
|
zcache_outstanding_writeback_pages =
|
|
|
atomic_inc_return(&zcache_outstanding_writeback_pages_atomic);
|
|
|
}
|
|
|
+static inline void dec_zcache_outstanding_writeback_pages(void)
|
|
|
+{
|
|
|
+ zcache_outstanding_writeback_pages =
|
|
|
+ atomic_dec_return(&zcache_outstanding_writeback_pages_atomic);
|
|
|
+};
|
|
|
static void unswiz(struct tmem_oid oid, u32 index,
|
|
|
unsigned *type, pgoff_t *offset);
|
|
|
|
|
@@ -1042,8 +1061,7 @@ static void unswiz(struct tmem_oid oid, u32 index,
|
|
|
static void zcache_end_swap_write(struct bio *bio, int err)
|
|
|
{
|
|
|
end_swap_bio_write(bio, err);
|
|
|
- zcache_outstanding_writeback_pages =
|
|
|
- atomic_dec_return(&zcache_outstanding_writeback_pages_atomic);
|
|
|
+ dec_zcache_outstanding_writeback_pages();
|
|
|
zcache_writtenback_pages++;
|
|
|
}
|
|
|
|