|
@@ -30,6 +30,7 @@ static const int cfq_slice_sync = HZ / 10;
|
|
|
static int cfq_slice_async = HZ / 25;
|
|
|
static const int cfq_slice_async_rq = 2;
|
|
|
static int cfq_slice_idle = HZ / 125;
|
|
|
+static int cfq_group_idle = HZ / 125;
|
|
|
static const int cfq_target_latency = HZ * 3/10; /* 300 ms */
|
|
|
static const int cfq_hist_divisor = 4;
|
|
|
|
|
@@ -147,6 +148,8 @@ struct cfq_queue {
|
|
|
struct cfq_queue *new_cfqq;
|
|
|
struct cfq_group *cfqg;
|
|
|
struct cfq_group *orig_cfqg;
|
|
|
+ /* Number of sectors dispatched from queue in single dispatch round */
|
|
|
+ unsigned long nr_sectors;
|
|
|
};
|
|
|
|
|
|
/*
|
|
@@ -198,6 +201,8 @@ struct cfq_group {
|
|
|
struct hlist_node cfqd_node;
|
|
|
atomic_t ref;
|
|
|
#endif
|
|
|
+ /* number of requests that are on the dispatch list or inside driver */
|
|
|
+ int dispatched;
|
|
|
};
|
|
|
|
|
|
/*
|
|
@@ -271,6 +276,7 @@ struct cfq_data {
|
|
|
unsigned int cfq_slice[2];
|
|
|
unsigned int cfq_slice_async_rq;
|
|
|
unsigned int cfq_slice_idle;
|
|
|
+ unsigned int cfq_group_idle;
|
|
|
unsigned int cfq_latency;
|
|
|
unsigned int cfq_group_isolation;
|
|
|
|
|
@@ -378,6 +384,21 @@ CFQ_CFQQ_FNS(wait_busy);
|
|
|
&cfqg->service_trees[i][j]: NULL) \
|
|
|
|
|
|
|
|
|
+static inline bool iops_mode(struct cfq_data *cfqd)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * If we are not idling on queues and it is a NCQ drive, parallel
|
|
|
+ * execution of requests is on and measuring time is not possible
|
|
|
+ * in most of the cases until and unless we drive shallower queue
|
|
|
+ * depths and that becomes a performance bottleneck. In such cases
|
|
|
+ * switch to start providing fairness in terms of number of IOs.
|
|
|
+ */
|
|
|
+ if (!cfqd->cfq_slice_idle && cfqd->hw_tag)
|
|
|
+ return true;
|
|
|
+ else
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq)
|
|
|
{
|
|
|
if (cfq_class_idle(cfqq))
|
|
@@ -906,7 +927,6 @@ static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq)
|
|
|
slice_used = cfqq->allocated_slice;
|
|
|
}
|
|
|
|
|
|
- cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u", slice_used);
|
|
|
return slice_used;
|
|
|
}
|
|
|
|
|
@@ -914,19 +934,21 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
|
|
|
struct cfq_queue *cfqq)
|
|
|
{
|
|
|
struct cfq_rb_root *st = &cfqd->grp_service_tree;
|
|
|
- unsigned int used_sl, charge_sl;
|
|
|
+ unsigned int used_sl, charge;
|
|
|
int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)
|
|
|
- cfqg->service_tree_idle.count;
|
|
|
|
|
|
BUG_ON(nr_sync < 0);
|
|
|
- used_sl = charge_sl = cfq_cfqq_slice_usage(cfqq);
|
|
|
+ used_sl = charge = cfq_cfqq_slice_usage(cfqq);
|
|
|
|
|
|
- if (!cfq_cfqq_sync(cfqq) && !nr_sync)
|
|
|
- charge_sl = cfqq->allocated_slice;
|
|
|
+ if (iops_mode(cfqd))
|
|
|
+ charge = cfqq->slice_dispatch;
|
|
|
+ else if (!cfq_cfqq_sync(cfqq) && !nr_sync)
|
|
|
+ charge = cfqq->allocated_slice;
|
|
|
|
|
|
/* Can't update vdisktime while group is on service tree */
|
|
|
cfq_rb_erase(&cfqg->rb_node, st);
|
|
|
- cfqg->vdisktime += cfq_scale_slice(charge_sl, cfqg);
|
|
|
+ cfqg->vdisktime += cfq_scale_slice(charge, cfqg);
|
|
|
__cfq_group_service_tree_add(st, cfqg);
|
|
|
|
|
|
/* This group is being expired. Save the context */
|
|
@@ -940,6 +962,9 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
|
|
|
|
|
|
cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
|
|
|
st->min_vdisktime);
|
|
|
+ cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u disp=%u charge=%u iops=%u"
|
|
|
+ " sect=%u", used_sl, cfqq->slice_dispatch, charge,
|
|
|
+ iops_mode(cfqd), cfqq->nr_sectors);
|
|
|
cfq_blkiocg_update_timeslice_used(&cfqg->blkg, used_sl);
|
|
|
cfq_blkiocg_set_start_empty_time(&cfqg->blkg);
|
|
|
}
|
|
@@ -1587,6 +1612,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
|
|
|
cfqq->allocated_slice = 0;
|
|
|
cfqq->slice_end = 0;
|
|
|
cfqq->slice_dispatch = 0;
|
|
|
+ cfqq->nr_sectors = 0;
|
|
|
|
|
|
cfq_clear_cfqq_wait_request(cfqq);
|
|
|
cfq_clear_cfqq_must_dispatch(cfqq);
|
|
@@ -1839,6 +1865,9 @@ static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
BUG_ON(!service_tree);
|
|
|
BUG_ON(!service_tree->count);
|
|
|
|
|
|
+ if (!cfqd->cfq_slice_idle)
|
|
|
+ return false;
|
|
|
+
|
|
|
/* We never do for idle class queues. */
|
|
|
if (prio == IDLE_WORKLOAD)
|
|
|
return false;
|
|
@@ -1863,7 +1892,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
|
|
|
{
|
|
|
struct cfq_queue *cfqq = cfqd->active_queue;
|
|
|
struct cfq_io_context *cic;
|
|
|
- unsigned long sl;
|
|
|
+ unsigned long sl, group_idle = 0;
|
|
|
|
|
|
/*
|
|
|
* SSD device without seek penalty, disable idling. But only do so
|
|
@@ -1879,8 +1908,13 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
|
|
|
/*
|
|
|
* idle is disabled, either manually or by past process history
|
|
|
*/
|
|
|
- if (!cfqd->cfq_slice_idle || !cfq_should_idle(cfqd, cfqq))
|
|
|
- return;
|
|
|
+ if (!cfq_should_idle(cfqd, cfqq)) {
|
|
|
+ /* no queue idling. Check for group idling */
|
|
|
+ if (cfqd->cfq_group_idle)
|
|
|
+ group_idle = cfqd->cfq_group_idle;
|
|
|
+ else
|
|
|
+ return;
|
|
|
+ }
|
|
|
|
|
|
/*
|
|
|
* still active requests from this queue, don't idle
|
|
@@ -1907,13 +1941,21 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
+ /* There are other queues in the group, don't do group idle */
|
|
|
+ if (group_idle && cfqq->cfqg->nr_cfqq > 1)
|
|
|
+ return;
|
|
|
+
|
|
|
cfq_mark_cfqq_wait_request(cfqq);
|
|
|
|
|
|
- sl = cfqd->cfq_slice_idle;
|
|
|
+ if (group_idle)
|
|
|
+ sl = cfqd->cfq_group_idle;
|
|
|
+ else
|
|
|
+ sl = cfqd->cfq_slice_idle;
|
|
|
|
|
|
mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
|
|
|
cfq_blkiocg_update_set_idle_time_stats(&cfqq->cfqg->blkg);
|
|
|
- cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu", sl);
|
|
|
+ cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl,
|
|
|
+ group_idle ? 1 : 0);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1929,9 +1971,11 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)
|
|
|
cfqq->next_rq = cfq_find_next_rq(cfqd, cfqq, rq);
|
|
|
cfq_remove_request(rq);
|
|
|
cfqq->dispatched++;
|
|
|
+ (RQ_CFQG(rq))->dispatched++;
|
|
|
elv_dispatch_sort(q, rq);
|
|
|
|
|
|
cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
|
|
|
+ cfqq->nr_sectors += blk_rq_sectors(rq);
|
|
|
cfq_blkiocg_update_dispatch_stats(&cfqq->cfqg->blkg, blk_rq_bytes(rq),
|
|
|
rq_data_dir(rq), rq_is_sync(rq));
|
|
|
}
|
|
@@ -2198,7 +2242,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
|
|
|
cfqq = NULL;
|
|
|
goto keep_queue;
|
|
|
} else
|
|
|
- goto expire;
|
|
|
+ goto check_group_idle;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -2226,8 +2270,23 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
|
|
|
* flight or is idling for a new request, allow either of these
|
|
|
* conditions to happen (or time out) before selecting a new queue.
|
|
|
*/
|
|
|
- if (timer_pending(&cfqd->idle_slice_timer) ||
|
|
|
- (cfqq->dispatched && cfq_should_idle(cfqd, cfqq))) {
|
|
|
+ if (timer_pending(&cfqd->idle_slice_timer)) {
|
|
|
+ cfqq = NULL;
|
|
|
+ goto keep_queue;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) {
|
|
|
+ cfqq = NULL;
|
|
|
+ goto keep_queue;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If group idle is enabled and there are requests dispatched from
|
|
|
+ * this group, wait for requests to complete.
|
|
|
+ */
|
|
|
+check_group_idle:
|
|
|
+ if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1
|
|
|
+ && cfqq->cfqg->dispatched) {
|
|
|
cfqq = NULL;
|
|
|
goto keep_queue;
|
|
|
}
|
|
@@ -3375,6 +3434,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
|
|
|
WARN_ON(!cfqq->dispatched);
|
|
|
cfqd->rq_in_driver--;
|
|
|
cfqq->dispatched--;
|
|
|
+ (RQ_CFQG(rq))->dispatched--;
|
|
|
cfq_blkiocg_update_completion_stats(&cfqq->cfqg->blkg,
|
|
|
rq_start_time_ns(rq), rq_io_start_time_ns(rq),
|
|
|
rq_data_dir(rq), rq_is_sync(rq));
|
|
@@ -3404,7 +3464,10 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
|
|
|
* the queue.
|
|
|
*/
|
|
|
if (cfq_should_wait_busy(cfqd, cfqq)) {
|
|
|
- cfqq->slice_end = jiffies + cfqd->cfq_slice_idle;
|
|
|
+ unsigned long extend_sl = cfqd->cfq_slice_idle;
|
|
|
+ if (!cfqd->cfq_slice_idle)
|
|
|
+ extend_sl = cfqd->cfq_group_idle;
|
|
|
+ cfqq->slice_end = jiffies + extend_sl;
|
|
|
cfq_mark_cfqq_wait_busy(cfqq);
|
|
|
cfq_log_cfqq(cfqd, cfqq, "will busy wait");
|
|
|
}
|
|
@@ -3850,6 +3913,7 @@ static void *cfq_init_queue(struct request_queue *q)
|
|
|
cfqd->cfq_slice[1] = cfq_slice_sync;
|
|
|
cfqd->cfq_slice_async_rq = cfq_slice_async_rq;
|
|
|
cfqd->cfq_slice_idle = cfq_slice_idle;
|
|
|
+ cfqd->cfq_group_idle = cfq_group_idle;
|
|
|
cfqd->cfq_latency = 1;
|
|
|
cfqd->cfq_group_isolation = 0;
|
|
|
cfqd->hw_tag = -1;
|
|
@@ -3922,6 +3986,7 @@ SHOW_FUNCTION(cfq_fifo_expire_async_show, cfqd->cfq_fifo_expire[0], 1);
|
|
|
SHOW_FUNCTION(cfq_back_seek_max_show, cfqd->cfq_back_max, 0);
|
|
|
SHOW_FUNCTION(cfq_back_seek_penalty_show, cfqd->cfq_back_penalty, 0);
|
|
|
SHOW_FUNCTION(cfq_slice_idle_show, cfqd->cfq_slice_idle, 1);
|
|
|
+SHOW_FUNCTION(cfq_group_idle_show, cfqd->cfq_group_idle, 1);
|
|
|
SHOW_FUNCTION(cfq_slice_sync_show, cfqd->cfq_slice[1], 1);
|
|
|
SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1);
|
|
|
SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0);
|
|
@@ -3954,6 +4019,7 @@ STORE_FUNCTION(cfq_back_seek_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0);
|
|
|
STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1,
|
|
|
UINT_MAX, 0);
|
|
|
STORE_FUNCTION(cfq_slice_idle_store, &cfqd->cfq_slice_idle, 0, UINT_MAX, 1);
|
|
|
+STORE_FUNCTION(cfq_group_idle_store, &cfqd->cfq_group_idle, 0, UINT_MAX, 1);
|
|
|
STORE_FUNCTION(cfq_slice_sync_store, &cfqd->cfq_slice[1], 1, UINT_MAX, 1);
|
|
|
STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1);
|
|
|
STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1,
|
|
@@ -3975,6 +4041,7 @@ static struct elv_fs_entry cfq_attrs[] = {
|
|
|
CFQ_ATTR(slice_async),
|
|
|
CFQ_ATTR(slice_async_rq),
|
|
|
CFQ_ATTR(slice_idle),
|
|
|
+ CFQ_ATTR(group_idle),
|
|
|
CFQ_ATTR(low_latency),
|
|
|
CFQ_ATTR(group_isolation),
|
|
|
__ATTR_NULL
|
|
@@ -4028,6 +4095,12 @@ static int __init cfq_init(void)
|
|
|
if (!cfq_slice_idle)
|
|
|
cfq_slice_idle = 1;
|
|
|
|
|
|
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
|
|
|
+ if (!cfq_group_idle)
|
|
|
+ cfq_group_idle = 1;
|
|
|
+#else
|
|
|
+ cfq_group_idle = 0;
|
|
|
+#endif
|
|
|
if (cfq_slab_setup())
|
|
|
return -ENOMEM;
|
|
|
|