@@ -1440,9 +1440,11 @@ IO
The "io" controller regulates the distribution of IO resources. This
controller implements both weight based and absolute bandwidth or IOPS
-limit distribution; however, weight based distribution is available
-only if cfq-iosched is in use and neither scheme is available for
-blk-mq devices.
+limit distribution. Weight based distribution is implemented by
+either iocost controller or bfq IO scheduler. When bfq is selected as
+the IO scheduler for any block device, iocost is disabled and bfq's
+implementation overrides for all devices. If bfq is built as a kernel
+module, unloading it re-enables iocost.
IO Interface Files
@@ -1194,7 +1194,9 @@ struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
}
struct blkcg_policy blkcg_policy_bfq = {
+#ifndef CONFIG_BLK_CGROUP_IOCOST
.dfl_cftypes = bfq_blkg_files,
+#endif
.legacy_cftypes = bfq_blkcg_legacy_files,
.cpd_alloc_fn = bfq_cpd_alloc,
@@ -6382,6 +6382,36 @@ static void bfq_init_root_group(struct bfq_group *root_group,
root_group->sched_data.bfq_class_idle_last_service = jiffies;
}
+#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_BLK_CGROUP_IOCOST)
+static bool bfq_enabled = false;
+
+static void bfq_enable(void)
+{
+ static DEFINE_MUTEX(bfq_enable_mutex);
+
+ mutex_lock(&bfq_enable_mutex);
+ if (!bfq_enabled) {
+ pr_info("bfq-iosched: Overriding iocost\n");
+ blkcg_policy_unregister(&blkcg_policy_iocost);
+ cgroup_add_dfl_cftypes(&io_cgrp_subsys, bfq_blkg_files);
+ bfq_enabled = true;
+ }
+ mutex_unlock(&bfq_enable_mutex);
+}
+
+static void __exit bfq_disable(void)
+{
+ if (bfq_enabled) {
+ pr_info("bfq-iosched: Restoring iocost\n");
+ cgroup_rm_cftypes(bfq_blkg_files);
+ blkcg_policy_register(&blkcg_policy_iocost);
+ }
+}
+#else
+static void bfq_enable(void) {}
+static void __exit bfq_disable(void) {}
+#endif
+
static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
{
struct bfq_data *bfqd;
@@ -6506,6 +6536,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group);
wbt_disable_default(q);
+ bfq_enable();
return 0;
out_free:
@@ -6823,6 +6854,7 @@ static void __exit bfq_exit(void)
blkcg_policy_unregister(&blkcg_policy_bfq);
#endif
bfq_slab_kill();
+ bfq_disable();
}
module_init(bfq_init);
@@ -605,8 +605,6 @@ static u32 vrate_adj_pct[] =
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 16 };
-static struct blkcg_policy blkcg_policy_iocost;
-
/* accessors and helpers */
static struct ioc *rqos_to_ioc(struct rq_qos *rqos)
{
@@ -2442,7 +2440,7 @@ static struct cftype ioc_files[] = {
{}
};
-static struct blkcg_policy blkcg_policy_iocost = {
+struct blkcg_policy blkcg_policy_iocost = {
.dfl_cftypes = ioc_files,
.cpd_alloc_fn = ioc_cpd_alloc,
.cpd_free_fn = ioc_cpd_free,
@@ -2450,6 +2448,7 @@ static struct blkcg_policy blkcg_policy_iocost = {
.pd_init_fn = ioc_pd_init,
.pd_free_fn = ioc_pd_free,
};
+EXPORT_SYMBOL_GPL(blkcg_policy_iocost);
static int __init ioc_init(void)
{
@@ -815,6 +815,11 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg)
void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay);
void blkcg_maybe_throttle_current(void);
+
+#ifdef CONFIG_BLK_CGROUP_IOCOST
+extern struct blkcg_policy blkcg_policy_iocost;
+#endif
+
#else /* CONFIG_BLK_CGROUP */
struct blkcg {
@@ -4059,6 +4059,7 @@ int cgroup_rm_cftypes(struct cftype *cfts)
mutex_unlock(&cgroup_mutex);
return ret;
}
+EXPORT_SYMBOL_GPL(cgroup_rm_cftypes);
/**
* cgroup_add_cftypes - add an array of cftypes to a subsystem
@@ -4115,6 +4116,7 @@ int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
cft->flags |= __CFTYPE_ONLY_ON_DFL;
return cgroup_add_cftypes(ss, cfts);
}
+EXPORT_SYMBOL_GPL(cgroup_add_dfl_cftypes);
/**
* cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies