diff options
Diffstat (limited to 'block/cfq-iosched.c')
| -rw-r--r-- | block/cfq-iosched.c | 65 |
1 files changed, 45 insertions, 20 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 019f28eea9df..84887e2eb210 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -82,6 +82,7 @@ struct cfq_rb_root { unsigned count; u64 min_vdisktime; struct rb_node *active; + unsigned total_weight; }; #define CFQ_RB_ROOT (struct cfq_rb_root) { RB_ROOT, NULL, 0, 0, } @@ -172,6 +173,8 @@ struct cfq_group { /* number of cfqq currently on this group */ int nr_cfqq; + /* Per group busy queus average. Useful for workload slice calc. */ + unsigned int busy_queues_avg[2]; /* * rr lists of queues with requests, onle rr for each priority class. * Counts are embedded in the cfq_rb_root @@ -188,6 +191,8 @@ struct cfq_data { /* Root service tree for cfq_groups */ struct cfq_rb_root grp_service_tree; struct cfq_group root_group; + /* Number of active cfq groups on group service tree */ + int nr_groups; /* * The priority currently being served @@ -206,7 +211,6 @@ struct cfq_data { struct rb_root prio_trees[CFQ_PRIO_LISTS]; unsigned int busy_queues; - unsigned int busy_queues_avg[2]; int rq_in_driver[2]; int sync_flight; @@ -354,10 +358,10 @@ static enum wl_type_t cfqq_type(struct cfq_queue *cfqq) return SYNC_WORKLOAD; } -static inline int cfq_busy_queues_wl(enum wl_prio_t wl, struct cfq_data *cfqd) +static inline int cfq_group_busy_queues_wl(enum wl_prio_t wl, + struct cfq_data *cfqd, + struct cfq_group *cfqg) { - struct cfq_group *cfqg = &cfqd->root_group; - if (wl == IDLE_WORKLOAD) return cfqg->service_tree_idle.count; @@ -489,18 +493,27 @@ static void update_min_vdisktime(struct cfq_rb_root *st) * to quickly follows sudden increases and decrease slowly */ -static inline unsigned cfq_get_avg_queues(struct cfq_data *cfqd, bool rt) +static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd, + struct cfq_group *cfqg, bool rt) { unsigned min_q, max_q; unsigned mult = cfq_hist_divisor - 1; unsigned round = cfq_hist_divisor / 2; - unsigned busy = cfq_busy_queues_wl(rt, cfqd); + unsigned busy = cfq_group_busy_queues_wl(rt, cfqd, cfqg); - min_q = min(cfqd->busy_queues_avg[rt], busy); - max_q = max(cfqd->busy_queues_avg[rt], busy); - cfqd->busy_queues_avg[rt] = (mult * max_q + min_q + round) / + min_q = min(cfqg->busy_queues_avg[rt], busy); + max_q = max(cfqg->busy_queues_avg[rt], busy); + cfqg->busy_queues_avg[rt] = (mult * max_q + min_q + round) / cfq_hist_divisor; - return cfqd->busy_queues_avg[rt]; + return cfqg->busy_queues_avg[rt]; +} + +static inline unsigned +cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg) +{ + struct cfq_rb_root *st = &cfqd->grp_service_tree; + + return cfq_target_latency * cfqg->weight / st->total_weight; } static inline void @@ -508,12 +521,17 @@ cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { unsigned slice = cfq_prio_to_slice(cfqd, cfqq); if (cfqd->cfq_latency) { - /* interested queues (we consider only the ones with the same - * priority class) */ - unsigned iq = cfq_get_avg_queues(cfqd, cfq_class_rt(cfqq)); + /* + * interested queues (we consider only the ones with the same + * priority class in the cfq group) + */ + unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, + cfq_class_rt(cfqq)); unsigned sync_slice = cfqd->cfq_slice[1]; unsigned expect_latency = sync_slice * iq; - if (expect_latency > cfq_target_latency) { + unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg); + + if (expect_latency > group_slice) { unsigned base_low_slice = 2 * cfqd->cfq_slice_idle; /* scale low_slice according to IO priority * and sync vs async */ @@ -521,7 +539,7 @@ cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) min(slice, base_low_slice * slice / sync_slice); /* the adapted slice value is scaled to fit all iqs * into the target latency */ - slice = max(slice * cfq_target_latency / expect_latency, + slice = max(slice * group_slice / expect_latency, low_slice); } } @@ -776,6 +794,8 @@ cfq_group_service_tree_add(struct cfq_data *cfqd, struct cfq_group *cfqg) __cfq_group_service_tree_add(st, cfqg); cfqg->on_st = true; + cfqd->nr_groups++; + st->total_weight += cfqg->weight; } static void @@ -794,6 +814,8 @@ cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg) return; cfqg->on_st = false; + cfqd->nr_groups--; + st->total_weight -= cfqg->weight; if (!RB_EMPTY_NODE(&cfqg->rb_node)) cfq_rb_erase(&cfqg->rb_node, st); } @@ -1639,6 +1661,7 @@ static void choose_service_tree(struct cfq_data *cfqd, struct cfq_group *cfqg) unsigned slice; unsigned count; struct cfq_rb_root *st; + unsigned group_slice; if (!cfqg) { cfqd->serving_prio = IDLE_WORKLOAD; @@ -1647,9 +1670,9 @@ static void choose_service_tree(struct cfq_data *cfqd, struct cfq_group *cfqg) } /* Choose next priority. RT > BE > IDLE */ - if (cfq_busy_queues_wl(RT_WORKLOAD, cfqd)) + if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg)) cfqd->serving_prio = RT_WORKLOAD; - else if (cfq_busy_queues_wl(BE_WORKLOAD, cfqd)) + else if (cfq_group_busy_queues_wl(BE_WORKLOAD, cfqd, cfqg)) cfqd->serving_prio = BE_WORKLOAD; else { cfqd->serving_prio = IDLE_WORKLOAD; @@ -1687,9 +1710,11 @@ static void choose_service_tree(struct cfq_data *cfqd, struct cfq_group *cfqg) * proportional to the number of queues in that workload, over * all the queues in the same priority class */ - slice = cfq_target_latency * count / - max_t(unsigned, cfqd->busy_queues_avg[cfqd->serving_prio], - cfq_busy_queues_wl(cfqd->serving_prio, cfqd)); + group_slice = cfq_group_slice(cfqd, cfqg); + + slice = group_slice * count / + max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_prio], + cfq_group_busy_queues_wl(cfqd->serving_prio, cfqd, cfqg)); if (cfqd->serving_type == ASYNC_WORKLOAD) /* async workload slice is scaled down according to |