[v2,2/4] QoS: Enhance framework to support per-cpu PM QoS request

Message ID 1407945689-18494-3-git-send-email-lina.iyer@linaro.org
State New
Headers show

Commit Message

Lina Iyer Aug. 13, 2014, 4:01 p.m.
QoS request can be better optimized if the request can be set only for
the required cpus and not all cpus. This helps save power on other
cores, while still gauranteeing the quality of service on the desired
cores.

Add a new enumeration to specify the PM QoS request type. The enums help
specify what is the intended target cpu of the request.

Enhance the QoS constraints data structures to support target value for
each core. Requests specify if the QoS is applicable to all cores
(default) or to a selective subset of the cores or to a core(s).

Idle and interested drivers can request a PM QoS value for a constraint
across all cpus, or a specific cpu or a set of cpus. Separate APIs have
been added to request for individual cpu or a cpumask.  The default
behaviour of PM QoS is maintained i.e, requests that do not specify a
type of the request will continue to be effected on all cores.

The userspace sysfs interface does not support setting cpumask of a PM
QoS request.

Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
Signed-off-by: Lina Iyer <lina.iyer@linaro.org>
---
 Documentation/power/pm_qos_interface.txt |  16 +++++
 include/linux/pm_qos.h                   |  13 ++++
 kernel/power/qos.c                       | 102 +++++++++++++++++++++++++++++++
 3 files changed, 131 insertions(+)

Comments

Javi Merino Aug. 15, 2014, 12:37 p.m. | #1
Hi Lina, some minor nits,

On Wed, Aug 13, 2014 at 05:01:27PM +0100, Lina Iyer wrote:
> QoS request can be better optimized if the request can be set only for
> the required cpus and not all cpus. This helps save power on other
> cores, while still gauranteeing the quality of service on the desired

                     guaranteeing

> cores.
> 
> Add a new enumeration to specify the PM QoS request type. The enums help
> specify what is the intended target cpu of the request.
> 
> Enhance the QoS constraints data structures to support target value for
> each core. Requests specify if the QoS is applicable to all cores
> (default) or to a selective subset of the cores or to a core(s).
> 
> Idle and interested drivers can request a PM QoS value for a constraint
> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
> been added to request for individual cpu or a cpumask.  The default
> behaviour of PM QoS is maintained i.e, requests that do not specify a
> type of the request will continue to be effected on all cores.
> 
> The userspace sysfs interface does not support setting cpumask of a PM
> QoS request.
> 
> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>
> ---
>  Documentation/power/pm_qos_interface.txt |  16 +++++
>  include/linux/pm_qos.h                   |  13 ++++
>  kernel/power/qos.c                       | 102 +++++++++++++++++++++++++++++++
>  3 files changed, 131 insertions(+)
> 
[...]
> diff --git a/kernel/power/qos.c b/kernel/power/qos.c
> index d0b9c0f..27f84a2 100644
> --- a/kernel/power/qos.c
> +++ b/kernel/power/qos.c
> @@ -65,6 +65,8 @@ static BLOCKING_NOTIFIER_HEAD(cpu_dma_lat_notifier);
>  static struct pm_qos_constraints cpu_dma_constraints = {
>  	.list = PLIST_HEAD_INIT(cpu_dma_constraints.list),
>  	.target_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
> +				PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE },
>  	.default_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
>  	.no_constraint_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
>  	.type = PM_QOS_MIN,
> @@ -79,6 +81,8 @@ static BLOCKING_NOTIFIER_HEAD(network_lat_notifier);
>  static struct pm_qos_constraints network_lat_constraints = {
>  	.list = PLIST_HEAD_INIT(network_lat_constraints.list),
>  	.target_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
> +				PM_QOS_NETWORK_LAT_DEFAULT_VALUE },
>  	.default_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
>  	.no_constraint_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
>  	.type = PM_QOS_MIN,
> @@ -94,6 +98,8 @@ static BLOCKING_NOTIFIER_HEAD(network_throughput_notifier);
>  static struct pm_qos_constraints network_tput_constraints = {
>  	.list = PLIST_HEAD_INIT(network_tput_constraints.list),
>  	.target_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
> +				PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE },
>  	.default_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
>  	.no_constraint_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
>  	.type = PM_QOS_MAX,
> @@ -157,6 +163,43 @@ static inline void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
>  	c->target_value = value;
>  }
>  
> +static inline void pm_qos_set_value_for_cpus(struct pm_qos_constraints *c)
> +{
> +	struct pm_qos_request *req = NULL;
> +	int cpu;
> +	s32 *qos_val;
> +
> +	qos_val = kcalloc(NR_CPUS, sizeof(*qos_val), GFP_KERNEL);
> +	if (!qos_val) {
> +		WARN("%s: No memory for PM QoS\n", __func__);
> +		return;
> +	}
> +
> +	for_each_possible_cpu(cpu)
> +		qos_val[cpu] = c->default_value;
> +
> +	plist_for_each_entry(req, &c->list, node) {
> +		for_each_cpu(cpu, &req->cpus_affine) {
> +			switch (c->type) {
> +			case PM_QOS_MIN:
> +				if (qos_val[cpu] > req->node.prio)
> +					qos_val[cpu] = req->node.prio;
> +				break;
> +			case PM_QOS_MAX:
> +				if (req->node.prio > qos_val[cpu])
> +					qos_val[cpu] = req->node.prio;
> +				break;
> +			default:
> +				BUG();
> +				break;
> +			}
> +		}
> +	}
> +
> +	for_each_possible_cpu(cpu)
> +		c->target_per_cpu[cpu] = qos_val[cpu];
> +}
> +
>  /**
>   * pm_qos_update_target - manages the constraints list and calls the notifiers
>   *  if needed
> @@ -206,6 +249,7 @@ int pm_qos_update_target(struct pm_qos_constraints *c,
>  
>  	curr_value = pm_qos_get_value(c);
>  	pm_qos_set_value(c, curr_value);
> +	pm_qos_set_value_for_cpus(c);
>  
>  	spin_unlock_irqrestore(&pm_qos_lock, flags);
>  
> @@ -298,6 +342,44 @@ int pm_qos_request(int pm_qos_class)
>  }
>  EXPORT_SYMBOL_GPL(pm_qos_request);
>  
> +int pm_qos_request_for_cpu(int pm_qos_class, int cpu)
> +{
> +	return pm_qos_array[pm_qos_class]->constraints->target_per_cpu[cpu];
> +}
> +EXPORT_SYMBOL(pm_qos_request_for_cpu);
> +
> +int pm_qos_request_for_cpumask(int pm_qos_class, struct cpumask *mask)
> +{
> +	unsigned long irqflags;
> +	int cpu;
> +	struct pm_qos_constraints *c = NULL;
> +	int val;
> +
> +	spin_lock_irqsave(&pm_qos_lock, irqflags);
> +	c = pm_qos_array[pm_qos_class]->constraints;
> +	val = c->default_value;
> +
> +	for_each_cpu(cpu, mask) {
> +		switch (c->type) {
> +		case PM_QOS_MIN:
> +			if (c->target_per_cpu[cpu] < val)
> +				val = c->target_per_cpu[cpu];
> +			break;
> +		case PM_QOS_MAX:
> +			if (c->target_per_cpu[cpu] > val)
> +				val = c->target_per_cpu[cpu];
> +			break;
> +		default:
> +			BUG();
> +			break;
> +		}
> +	}
> +	spin_unlock_irqrestore(&pm_qos_lock, irqflags);
> +
> +	return val;
> +}
> +EXPORT_SYMBOL(pm_qos_request_for_cpumask);
> +
>  int pm_qos_request_active(struct pm_qos_request *req)
>  {
>  	return req->pm_qos_class != 0;
> @@ -353,6 +435,24 @@ void pm_qos_add_request(struct pm_qos_request *req,
>  		WARN(1, KERN_ERR "pm_qos_add_request() called for already added request\n");
>  		return;
>  	}
> +
> +	switch (req->type) {
> +	case PM_QOS_REQ_AFFINE_CORES:
> +		if (cpumask_empty(&req->cpus_affine)) {
> +			req->type = PM_QOS_REQ_ALL_CORES;
> +			cpumask_setall(&req->cpus_affine);
> +			WARN(1, KERN_ERR "Affine cores not set for request with affinity flag\n");
> +		}
> +		break;
> +
> +	default:
> +		WARN(1, KERN_ERR "Unknown request type %d\n", req->type);
> +		/* fall through */
> +	case PM_QOS_REQ_ALL_CORES:
> +		cpumask_setall(&req->cpus_affine);
> +		break;
> +	}
> +
>  	req->pm_qos_class = pm_qos_class;
>  	INIT_DELAYED_WORK(&req->work, pm_qos_work_fn);
>  	trace_pm_qos_add_request(pm_qos_class, value);
> @@ -426,6 +526,7 @@ void pm_qos_update_request_timeout(struct pm_qos_request *req, s32 new_value,
>   */
>  void pm_qos_remove_request(struct pm_qos_request *req)
>  {
> +

Unnecessary newline added.

>  	if (!req) /*guard against callers passing in null */
>  		return;
>  		/* silent return to keep pcm code cleaner */
> @@ -441,6 +542,7 @@ void pm_qos_remove_request(struct pm_qos_request *req)
>  	pm_qos_update_target(pm_qos_array[req->pm_qos_class]->constraints,
>  			     req, PM_QOS_REMOVE_REQ,
>  			     PM_QOS_DEFAULT_VALUE);
> +

ditto.  Cheers,
Javi

>  	memset(req, 0, sizeof(*req));
>  }
>  EXPORT_SYMBOL_GPL(pm_qos_remove_request);
> -- 
> 1.9.1
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-pm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Lina Iyer Aug. 15, 2014, 3:06 p.m. | #2
On Fri, Aug 15, 2014 at 01:37:32PM +0100, Javi Merino wrote:
>Hi Lina, some minor nits,
>
>On Wed, Aug 13, 2014 at 05:01:27PM +0100, Lina Iyer wrote:
>> QoS request can be better optimized if the request can be set only for
>> the required cpus and not all cpus. This helps save power on other
>> cores, while still gauranteeing the quality of service on the desired
>
>                     guaranteeing
>
I never get that right :[
Will take care of it.
>> cores.
>>
>> Add a new enumeration to specify the PM QoS request type. The enums help
>> specify what is the intended target cpu of the request.
>>
>> Enhance the QoS constraints data structures to support target value for
>> each core. Requests specify if the QoS is applicable to all cores
>> (default) or to a selective subset of the cores or to a core(s).
>>
>> Idle and interested drivers can request a PM QoS value for a constraint
>> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
>> been added to request for individual cpu or a cpumask.  The default
>> behaviour of PM QoS is maintained i.e, requests that do not specify a
>> type of the request will continue to be effected on all cores.
>>
>> The userspace sysfs interface does not support setting cpumask of a PM
>> QoS request.
>>
>> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
>> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>
>> ---
>>  Documentation/power/pm_qos_interface.txt |  16 +++++
>>  include/linux/pm_qos.h                   |  13 ++++
>>  kernel/power/qos.c                       | 102 +++++++++++++++++++++++++++++++
>>  3 files changed, 131 insertions(+)
>>
>[...]
>> diff --git a/kernel/power/qos.c b/kernel/power/qos.c
>> index d0b9c0f..27f84a2 100644
>> --- a/kernel/power/qos.c
>> +++ b/kernel/power/qos.c
>> @@ -65,6 +65,8 @@ static BLOCKING_NOTIFIER_HEAD(cpu_dma_lat_notifier);
>>  static struct pm_qos_constraints cpu_dma_constraints = {
>>  	.list = PLIST_HEAD_INIT(cpu_dma_constraints.list),
>>  	.target_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
>> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
>> +				PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE },
>>  	.default_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
>>  	.no_constraint_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
>>  	.type = PM_QOS_MIN,
>> @@ -79,6 +81,8 @@ static BLOCKING_NOTIFIER_HEAD(network_lat_notifier);
>>  static struct pm_qos_constraints network_lat_constraints = {
>>  	.list = PLIST_HEAD_INIT(network_lat_constraints.list),
>>  	.target_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
>> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
>> +				PM_QOS_NETWORK_LAT_DEFAULT_VALUE },
>>  	.default_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
>>  	.no_constraint_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
>>  	.type = PM_QOS_MIN,
>> @@ -94,6 +98,8 @@ static BLOCKING_NOTIFIER_HEAD(network_throughput_notifier);
>>  static struct pm_qos_constraints network_tput_constraints = {
>>  	.list = PLIST_HEAD_INIT(network_tput_constraints.list),
>>  	.target_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
>> +	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
>> +				PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE },
>>  	.default_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
>>  	.no_constraint_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
>>  	.type = PM_QOS_MAX,
>> @@ -157,6 +163,43 @@ static inline void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
>>  	c->target_value = value;
>>  }
>>
>> +static inline void pm_qos_set_value_for_cpus(struct pm_qos_constraints *c)
>> +{
>> +	struct pm_qos_request *req = NULL;
>> +	int cpu;
>> +	s32 *qos_val;
>> +
>> +	qos_val = kcalloc(NR_CPUS, sizeof(*qos_val), GFP_KERNEL);
>> +	if (!qos_val) {
>> +		WARN("%s: No memory for PM QoS\n", __func__);
>> +		return;
>> +	}
>> +
>> +	for_each_possible_cpu(cpu)
>> +		qos_val[cpu] = c->default_value;
>> +
>> +	plist_for_each_entry(req, &c->list, node) {
>> +		for_each_cpu(cpu, &req->cpus_affine) {
>> +			switch (c->type) {
>> +			case PM_QOS_MIN:
>> +				if (qos_val[cpu] > req->node.prio)
>> +					qos_val[cpu] = req->node.prio;
>> +				break;
>> +			case PM_QOS_MAX:
>> +				if (req->node.prio > qos_val[cpu])
>> +					qos_val[cpu] = req->node.prio;
>> +				break;
>> +			default:
>> +				BUG();
>> +				break;
>> +			}
>> +		}
>> +	}
>> +
>> +	for_each_possible_cpu(cpu)
>> +		c->target_per_cpu[cpu] = qos_val[cpu];
>> +}
>> +
>>  /**
>>   * pm_qos_update_target - manages the constraints list and calls the notifiers
>>   *  if needed
>> @@ -206,6 +249,7 @@ int pm_qos_update_target(struct pm_qos_constraints *c,
>>
>>  	curr_value = pm_qos_get_value(c);
>>  	pm_qos_set_value(c, curr_value);
>> +	pm_qos_set_value_for_cpus(c);
>>
>>  	spin_unlock_irqrestore(&pm_qos_lock, flags);
>>
>> @@ -298,6 +342,44 @@ int pm_qos_request(int pm_qos_class)
>>  }
>>  EXPORT_SYMBOL_GPL(pm_qos_request);
>>
>> +int pm_qos_request_for_cpu(int pm_qos_class, int cpu)
>> +{
>> +	return pm_qos_array[pm_qos_class]->constraints->target_per_cpu[cpu];
>> +}
>> +EXPORT_SYMBOL(pm_qos_request_for_cpu);
>> +
>> +int pm_qos_request_for_cpumask(int pm_qos_class, struct cpumask *mask)
>> +{
>> +	unsigned long irqflags;
>> +	int cpu;
>> +	struct pm_qos_constraints *c = NULL;
>> +	int val;
>> +
>> +	spin_lock_irqsave(&pm_qos_lock, irqflags);
>> +	c = pm_qos_array[pm_qos_class]->constraints;
>> +	val = c->default_value;
>> +
>> +	for_each_cpu(cpu, mask) {
>> +		switch (c->type) {
>> +		case PM_QOS_MIN:
>> +			if (c->target_per_cpu[cpu] < val)
>> +				val = c->target_per_cpu[cpu];
>> +			break;
>> +		case PM_QOS_MAX:
>> +			if (c->target_per_cpu[cpu] > val)
>> +				val = c->target_per_cpu[cpu];
>> +			break;
>> +		default:
>> +			BUG();
>> +			break;
>> +		}
>> +	}
>> +	spin_unlock_irqrestore(&pm_qos_lock, irqflags);
>> +
>> +	return val;
>> +}
>> +EXPORT_SYMBOL(pm_qos_request_for_cpumask);
>> +
>>  int pm_qos_request_active(struct pm_qos_request *req)
>>  {
>>  	return req->pm_qos_class != 0;
>> @@ -353,6 +435,24 @@ void pm_qos_add_request(struct pm_qos_request *req,
>>  		WARN(1, KERN_ERR "pm_qos_add_request() called for already added request\n");
>>  		return;
>>  	}
>> +
>> +	switch (req->type) {
>> +	case PM_QOS_REQ_AFFINE_CORES:
>> +		if (cpumask_empty(&req->cpus_affine)) {
>> +			req->type = PM_QOS_REQ_ALL_CORES;
>> +			cpumask_setall(&req->cpus_affine);
>> +			WARN(1, KERN_ERR "Affine cores not set for request with affinity flag\n");
>> +		}
>> +		break;
>> +
>> +	default:
>> +		WARN(1, KERN_ERR "Unknown request type %d\n", req->type);
>> +		/* fall through */
>> +	case PM_QOS_REQ_ALL_CORES:
>> +		cpumask_setall(&req->cpus_affine);
>> +		break;
>> +	}
>> +
>>  	req->pm_qos_class = pm_qos_class;
>>  	INIT_DELAYED_WORK(&req->work, pm_qos_work_fn);
>>  	trace_pm_qos_add_request(pm_qos_class, value);
>> @@ -426,6 +526,7 @@ void pm_qos_update_request_timeout(struct pm_qos_request *req, s32 new_value,
>>   */
>>  void pm_qos_remove_request(struct pm_qos_request *req)
>>  {
>> +
>
>Unnecessary newline added.
>
>>  	if (!req) /*guard against callers passing in null */
>>  		return;
>>  		/* silent return to keep pcm code cleaner */
>> @@ -441,6 +542,7 @@ void pm_qos_remove_request(struct pm_qos_request *req)
>>  	pm_qos_update_target(pm_qos_array[req->pm_qos_class]->constraints,
>>  			     req, PM_QOS_REMOVE_REQ,
>>  			     PM_QOS_DEFAULT_VALUE);
>> +
>
>ditto.  Cheers,
>Javi
>
>>  	memset(req, 0, sizeof(*req));
>>  }
>>  EXPORT_SYMBOL_GPL(pm_qos_remove_request);
>> --
>> 1.9.1
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-pm" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
>
--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Kevin Hilman Aug. 18, 2014, 11:55 p.m. | #3
Hi Lina,

Lina Iyer <lina.iyer@linaro.org> writes:

> QoS request can be better optimized if the request can be set only for
> the required cpus and not all cpus. This helps save power on other
> cores, while still gauranteeing the quality of service on the desired
> cores.
>
> Add a new enumeration to specify the PM QoS request type. The enums help
> specify what is the intended target cpu of the request.
>
> Enhance the QoS constraints data structures to support target value for
> each core. Requests specify if the QoS is applicable to all cores
> (default) or to a selective subset of the cores or to a core(s).
>
> Idle and interested drivers can request a PM QoS value for a constraint
> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
> been added to request for individual cpu or a cpumask.  The default
> behaviour of PM QoS is maintained i.e, requests that do not specify a
> type of the request will continue to be effected on all cores.
>
> The userspace sysfs interface does not support setting cpumask of a PM
> QoS request.
>
> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>

I agree this is a needed feature.  I didn't study it in detail yet, but
after a quick glance, it looks like a good approach.  

However, I did start to wonder how this will behave in the context of
the hotplug.  For example, what if a constraint is setup with a cpumask,
then one of those CPUs is hotplugged away.  

Kevin
--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Lina Iyer Aug. 19, 2014, 12:34 a.m. | #4
On Mon, Aug 18, 2014 at 04:55:41PM -0700, Kevin Hilman wrote:
>Hi Lina,
>
>Lina Iyer <lina.iyer@linaro.org> writes:
>
>> QoS request can be better optimized if the request can be set only for
>> the required cpus and not all cpus. This helps save power on other
>> cores, while still gauranteeing the quality of service on the desired
>> cores.
>>
>> Add a new enumeration to specify the PM QoS request type. The enums help
>> specify what is the intended target cpu of the request.
>>
>> Enhance the QoS constraints data structures to support target value for
>> each core. Requests specify if the QoS is applicable to all cores
>> (default) or to a selective subset of the cores or to a core(s).
>>
>> Idle and interested drivers can request a PM QoS value for a constraint
>> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
>> been added to request for individual cpu or a cpumask.  The default
>> behaviour of PM QoS is maintained i.e, requests that do not specify a
>> type of the request will continue to be effected on all cores.
>>
>> The userspace sysfs interface does not support setting cpumask of a PM
>> QoS request.
>>
>> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
>> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>
>
>I agree this is a needed feature.  I didn't study it in detail yet, but
>after a quick glance, it looks like a good approach.
>
>However, I did start to wonder how this will behave in the context of
>the hotplug.  For example, what if a constraint is setup with a cpumask,
>then one of those CPUs is hotplugged away.
>
Thanks for bringing it up. I forgot to mention this in the series, but
well, it can be addressed.

When a core is hotplugged, the IRQ migrates to the next online in the
smp_affinity. The QoS code would work in the hotplug case as well, with
a simple change. The current code does not send affinity_notifications
correctly because of a direct call to irq_chip->irq_set_affinity()
instead of calling the generic irq affinity api.

This is the simple change that needs to be made. I will submit a patch
for that.

In arm64/kernel/irq.c 

-       c = irq_data_get_irq_chip(d);
-       if (!c->irq_set_affinity)
-               pr_debug("IRQ%u: unable to set affinity\n", d->irq);
-       else if (c->irq_set_affinity(d, affinity, true) == IRQ_SET_MASK_OK && ret)
-               cpumask_copy(d->affinity, affinity);
-
-       return ret;
+       return __irq_set_affinity_locked(d, affinity) == 0;



>Kevin
--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Kevin Hilman Aug. 27, 2014, 6:01 p.m. | #5
Lina Iyer <lina.iyer@linaro.org> writes:

> QoS request can be better optimized if the request can be set only for
> the required cpus and not all cpus. This helps save power on other
> cores, while still gauranteeing the quality of service on the desired
> cores.
>
> Add a new enumeration to specify the PM QoS request type. The enums help
> specify what is the intended target cpu of the request.
>
> Enhance the QoS constraints data structures to support target value for
> each core. Requests specify if the QoS is applicable to all cores
> (default) or to a selective subset of the cores or to a core(s).
>
> Idle and interested drivers can request a PM QoS value for a constraint
> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
> been added to request for individual cpu or a cpumask.  The default
> behaviour of PM QoS is maintained i.e, requests that do not specify a
> type of the request will continue to be effected on all cores.
>
> The userspace sysfs interface does not support setting cpumask of a PM
> QoS request.
>
> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>

I'm curious if you looked at using the per-device QoS API for this
instead of expending the system-wide API.  IOW, from a per-device QoS
POV, a CPU is no different than any other device, and since we already
have the per-device QoS API, I wondered if that might be a better choice
to implment this per-CPU feature.

Kevin
--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Lina Iyer Aug. 27, 2014, 8:13 p.m. | #6
On Wed, Aug 27, 2014 at 11:01:40AM -0700, Kevin Hilman wrote:
>Lina Iyer <lina.iyer@linaro.org> writes:
>
>> QoS request can be better optimized if the request can be set only for
>> the required cpus and not all cpus. This helps save power on other
>> cores, while still gauranteeing the quality of service on the desired
>> cores.
>>
>> Add a new enumeration to specify the PM QoS request type. The enums help
>> specify what is the intended target cpu of the request.
>>
>> Enhance the QoS constraints data structures to support target value for
>> each core. Requests specify if the QoS is applicable to all cores
>> (default) or to a selective subset of the cores or to a core(s).
>>
>> Idle and interested drivers can request a PM QoS value for a constraint
>> across all cpus, or a specific cpu or a set of cpus. Separate APIs have
>> been added to request for individual cpu or a cpumask.  The default
>> behaviour of PM QoS is maintained i.e, requests that do not specify a
>> type of the request will continue to be effected on all cores.
>>
>> The userspace sysfs interface does not support setting cpumask of a PM
>> QoS request.
>>
>> Signed-off-by: Praveen Chidambaram <pchidamb@codeaurora.org>
>> Signed-off-by: Lina Iyer <lina.iyer@linaro.org>
>
>I'm curious if you looked at using the per-device QoS API for this
>instead of expending the system-wide API.  IOW, from a per-device QoS
>POV, a CPU is no different than any other device, and since we already
>have the per-device QoS API, I wondered if that might be a better choice
>to implment this per-CPU feature.
>
If you mean dev-pm-qos, then yes. I explored using that. The dev-pm-qos
is an user of the qos framework and holds an pm-qos object, but not in
any other way influences the final value of the QoS constraint other
than specify a request on behalf of the device.  IMHO, What we want is
complementary. When a device specifies a request, we want the request to
be directed at a set of cpus and that is a function of the QoS
framework, hence addressed by these patches to the QoS framework.

>Kevin
>--
>To unsubscribe from this list: send the line "unsubscribe linux-pm" in
>the body of a message to majordomo@vger.kernel.org
>More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-pm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Patch

diff --git a/Documentation/power/pm_qos_interface.txt b/Documentation/power/pm_qos_interface.txt
index a5da5c7..c129517 100644
--- a/Documentation/power/pm_qos_interface.txt
+++ b/Documentation/power/pm_qos_interface.txt
@@ -41,6 +41,15 @@  registered notifiers are called only if the target value is now different.
 Clients of pm_qos need to save the returned handle for future use in other
 pm_qos API functions.
 
+The handle is a pm_qos_request object. By default the request object sets the
+request type to PM_QOS_REQ_ALL_CORES, in which case, the PM QoS request
+applies to all cores. However, the driver can also specify a request type to
+be either of
+        PM_QOS_REQ_ALL_CORES,
+        PM_QOS_REQ_AFFINE_CORES,
+
+Specify the cpumask when type is set to PM_QOS_REQ_AFFINE_CORES.
+
 void pm_qos_update_request(handle, new_target_value):
 Will update the list element pointed to by the handle with the new target value
 and recompute the new aggregated target, calling the notification tree if the
@@ -54,6 +63,13 @@  the request.
 int pm_qos_request(param_class):
 Returns the aggregated value for a given PM QoS class.
 
+int pm_qos_request_for_cpu(param_class, cpu):
+Returns the aggregated value for a given PM QoS class for the specified cpu.
+
+int pm_qos_request_for_cpumask(param_class, cpumask):
+Returns the aggregated value for a given PM QoS class for the specified
+cpumask.
+
 int pm_qos_request_active(handle):
 Returns if the request is still active, i.e. it has not been removed from a
 PM QoS class constraints list.
diff --git a/include/linux/pm_qos.h b/include/linux/pm_qos.h
index e1b763d..a3aa5b5 100644
--- a/include/linux/pm_qos.h
+++ b/include/linux/pm_qos.h
@@ -9,6 +9,8 @@ 
 #include <linux/miscdevice.h>
 #include <linux/device.h>
 #include <linux/workqueue.h>
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
 
 enum {
 	PM_QOS_RESERVED = 0,
@@ -40,7 +42,15 @@  enum pm_qos_flags_status {
 #define PM_QOS_FLAG_NO_POWER_OFF	(1 << 0)
 #define PM_QOS_FLAG_REMOTE_WAKEUP	(1 << 1)
 
+enum pm_qos_req_type {
+	PM_QOS_REQ_ALL_CORES = 0,
+	PM_QOS_REQ_AFFINE_CORES,
+};
+
 struct pm_qos_request {
+	enum pm_qos_req_type type;
+	struct cpumask cpus_affine;
+	/* Internal structure members */
 	struct plist_node node;
 	int pm_qos_class;
 	struct delayed_work work; /* for pm_qos_update_request_timeout */
@@ -80,6 +90,7 @@  enum pm_qos_type {
 struct pm_qos_constraints {
 	struct plist_head list;
 	s32 target_value;	/* Do not change to 64 bit */
+	s32 target_per_cpu[NR_CPUS];
 	s32 default_value;
 	s32 no_constraint_value;
 	enum pm_qos_type type;
@@ -127,6 +138,8 @@  void pm_qos_update_request_timeout(struct pm_qos_request *req,
 void pm_qos_remove_request(struct pm_qos_request *req);
 
 int pm_qos_request(int pm_qos_class);
+int pm_qos_request_for_cpu(int pm_qos_class, int cpu);
+int pm_qos_request_for_cpumask(int pm_qos_class, struct cpumask *mask);
 int pm_qos_add_notifier(int pm_qos_class, struct notifier_block *notifier);
 int pm_qos_remove_notifier(int pm_qos_class, struct notifier_block *notifier);
 int pm_qos_request_active(struct pm_qos_request *req);
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index d0b9c0f..27f84a2 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -65,6 +65,8 @@  static BLOCKING_NOTIFIER_HEAD(cpu_dma_lat_notifier);
 static struct pm_qos_constraints cpu_dma_constraints = {
 	.list = PLIST_HEAD_INIT(cpu_dma_constraints.list),
 	.target_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
+	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
+				PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE },
 	.default_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
 	.no_constraint_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
 	.type = PM_QOS_MIN,
@@ -79,6 +81,8 @@  static BLOCKING_NOTIFIER_HEAD(network_lat_notifier);
 static struct pm_qos_constraints network_lat_constraints = {
 	.list = PLIST_HEAD_INIT(network_lat_constraints.list),
 	.target_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
+	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
+				PM_QOS_NETWORK_LAT_DEFAULT_VALUE },
 	.default_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
 	.no_constraint_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
 	.type = PM_QOS_MIN,
@@ -94,6 +98,8 @@  static BLOCKING_NOTIFIER_HEAD(network_throughput_notifier);
 static struct pm_qos_constraints network_tput_constraints = {
 	.list = PLIST_HEAD_INIT(network_tput_constraints.list),
 	.target_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
+	.target_per_cpu = { [0 ... (NR_CPUS - 1)] =
+				PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE },
 	.default_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
 	.no_constraint_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
 	.type = PM_QOS_MAX,
@@ -157,6 +163,43 @@  static inline void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
 	c->target_value = value;
 }
 
+static inline void pm_qos_set_value_for_cpus(struct pm_qos_constraints *c)
+{
+	struct pm_qos_request *req = NULL;
+	int cpu;
+	s32 *qos_val;
+
+	qos_val = kcalloc(NR_CPUS, sizeof(*qos_val), GFP_KERNEL);
+	if (!qos_val) {
+		WARN("%s: No memory for PM QoS\n", __func__);
+		return;
+	}
+
+	for_each_possible_cpu(cpu)
+		qos_val[cpu] = c->default_value;
+
+	plist_for_each_entry(req, &c->list, node) {
+		for_each_cpu(cpu, &req->cpus_affine) {
+			switch (c->type) {
+			case PM_QOS_MIN:
+				if (qos_val[cpu] > req->node.prio)
+					qos_val[cpu] = req->node.prio;
+				break;
+			case PM_QOS_MAX:
+				if (req->node.prio > qos_val[cpu])
+					qos_val[cpu] = req->node.prio;
+				break;
+			default:
+				BUG();
+				break;
+			}
+		}
+	}
+
+	for_each_possible_cpu(cpu)
+		c->target_per_cpu[cpu] = qos_val[cpu];
+}
+
 /**
  * pm_qos_update_target - manages the constraints list and calls the notifiers
  *  if needed
@@ -206,6 +249,7 @@  int pm_qos_update_target(struct pm_qos_constraints *c,
 
 	curr_value = pm_qos_get_value(c);
 	pm_qos_set_value(c, curr_value);
+	pm_qos_set_value_for_cpus(c);
 
 	spin_unlock_irqrestore(&pm_qos_lock, flags);
 
@@ -298,6 +342,44 @@  int pm_qos_request(int pm_qos_class)
 }
 EXPORT_SYMBOL_GPL(pm_qos_request);
 
+int pm_qos_request_for_cpu(int pm_qos_class, int cpu)
+{
+	return pm_qos_array[pm_qos_class]->constraints->target_per_cpu[cpu];
+}
+EXPORT_SYMBOL(pm_qos_request_for_cpu);
+
+int pm_qos_request_for_cpumask(int pm_qos_class, struct cpumask *mask)
+{
+	unsigned long irqflags;
+	int cpu;
+	struct pm_qos_constraints *c = NULL;
+	int val;
+
+	spin_lock_irqsave(&pm_qos_lock, irqflags);
+	c = pm_qos_array[pm_qos_class]->constraints;
+	val = c->default_value;
+
+	for_each_cpu(cpu, mask) {
+		switch (c->type) {
+		case PM_QOS_MIN:
+			if (c->target_per_cpu[cpu] < val)
+				val = c->target_per_cpu[cpu];
+			break;
+		case PM_QOS_MAX:
+			if (c->target_per_cpu[cpu] > val)
+				val = c->target_per_cpu[cpu];
+			break;
+		default:
+			BUG();
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&pm_qos_lock, irqflags);
+
+	return val;
+}
+EXPORT_SYMBOL(pm_qos_request_for_cpumask);
+
 int pm_qos_request_active(struct pm_qos_request *req)
 {
 	return req->pm_qos_class != 0;
@@ -353,6 +435,24 @@  void pm_qos_add_request(struct pm_qos_request *req,
 		WARN(1, KERN_ERR "pm_qos_add_request() called for already added request\n");
 		return;
 	}
+
+	switch (req->type) {
+	case PM_QOS_REQ_AFFINE_CORES:
+		if (cpumask_empty(&req->cpus_affine)) {
+			req->type = PM_QOS_REQ_ALL_CORES;
+			cpumask_setall(&req->cpus_affine);
+			WARN(1, KERN_ERR "Affine cores not set for request with affinity flag\n");
+		}
+		break;
+
+	default:
+		WARN(1, KERN_ERR "Unknown request type %d\n", req->type);
+		/* fall through */
+	case PM_QOS_REQ_ALL_CORES:
+		cpumask_setall(&req->cpus_affine);
+		break;
+	}
+
 	req->pm_qos_class = pm_qos_class;
 	INIT_DELAYED_WORK(&req->work, pm_qos_work_fn);
 	trace_pm_qos_add_request(pm_qos_class, value);
@@ -426,6 +526,7 @@  void pm_qos_update_request_timeout(struct pm_qos_request *req, s32 new_value,
  */
 void pm_qos_remove_request(struct pm_qos_request *req)
 {
+
 	if (!req) /*guard against callers passing in null */
 		return;
 		/* silent return to keep pcm code cleaner */
@@ -441,6 +542,7 @@  void pm_qos_remove_request(struct pm_qos_request *req)
 	pm_qos_update_target(pm_qos_array[req->pm_qos_class]->constraints,
 			     req, PM_QOS_REMOVE_REQ,
 			     PM_QOS_DEFAULT_VALUE);
+
 	memset(req, 0, sizeof(*req));
 }
 EXPORT_SYMBOL_GPL(pm_qos_remove_request);