[v2,4/4] PM: domains: Drop/restore performance state votes for devices at system PM

Message ID 20210603093438.138705-5-ulf.hansson@linaro.org
State New
Headers show
Series
  • PM: domains: Avoid boilerplate code for DVFS in subsystem/drivers
Related show

Commit Message

Ulf Hansson June 3, 2021, 9:34 a.m.
Recent changes in genpd drops and restore performance state votes for
devices during runtime PM.

For the similar reasons, but to avoid the same kind of boilerplate code in
device PM callbacks for system sleep in subsystems/drivers, let's drop and
restore performance states votes in genpd for the attached devices during
system sleep.

Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>

---

Changes in v2:
	- Rebased.
	- A few cosmetic changes.

---
 drivers/base/power/domain.c | 9 +++++++++
 include/linux/pm_domain.h   | 1 +
 2 files changed, 10 insertions(+)

-- 
2.25.1

Comments

Ulf Hansson June 3, 2021, 10:20 a.m. | #1
+ Mark Brown, Dmitry Baryshkov

On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:
>

> Recent changes in genpd drops and restore performance state votes for

> devices during runtime PM.

>

> For the similar reasons, but to avoid the same kind of boilerplate code in

> device PM callbacks for system sleep in subsystems/drivers, let's drop and

> restore performance states votes in genpd for the attached devices during

> system sleep.

>

> Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>


After a second thought, it looks like we maybe should defer to apply
this final patch of the series. At least until we figured out how to
address the below issue:

So, I noticed that we have things like "regulator-fixed-domain", that
uses "required-opps" to enable/disable a regulator through the
dev_pm_set_performance_state() interface. We likely don't want to drop
the performance state internally in genpd when genpd_suspend_noirq()
gets called, for the corresponding struct device for the regulator.

I guess if genpd should drop performance states like $subject patch
suggest, we need some kind of additional coordination, that allows a
subsystem/driver to inform genpd when it should avoid it. Or something
along those lines.

Kind regards
Uffe

> ---

>

> Changes in v2:

>         - Rebased.

>         - A few cosmetic changes.

>

> ---

>  drivers/base/power/domain.c | 9 +++++++++

>  include/linux/pm_domain.h   | 1 +

>  2 files changed, 10 insertions(+)

>

> diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c

> index e5d97174c254..a33e5b341f3f 100644

> --- a/drivers/base/power/domain.c

> +++ b/drivers/base/power/domain.c

> @@ -1171,6 +1171,7 @@ static int genpd_prepare(struct device *dev)

>   */

>  static int genpd_finish_suspend(struct device *dev, bool poweroff)

>  {

> +       struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev);

>         struct generic_pm_domain *genpd;

>         int ret = 0;

>

> @@ -1201,6 +1202,7 @@ static int genpd_finish_suspend(struct device *dev, bool poweroff)

>         }

>

>         genpd_lock(genpd);

> +       gpd_data->pm_pstate = genpd_drop_performance_state(dev);

>         genpd->suspended_count++;

>         genpd_sync_power_off(genpd, true, 0);

>         genpd_unlock(genpd);

> @@ -1245,6 +1247,7 @@ static int genpd_resume_noirq(struct device *dev)

>         genpd_lock(genpd);

>         genpd_sync_power_on(genpd, true, 0);

>         genpd->suspended_count--;

> +       genpd_restore_performance_state(dev, dev_gpd_data(dev)->pm_pstate);

>         genpd_unlock(genpd);

>

>         if (genpd->dev_ops.stop && genpd->dev_ops.start &&

> @@ -1364,6 +1367,7 @@ static int genpd_restore_noirq(struct device *dev)

>         }

>

>         genpd_sync_power_on(genpd, true, 0);

> +       genpd_restore_performance_state(dev, dev_gpd_data(dev)->pm_pstate);

>         genpd_unlock(genpd);

>

>         if (genpd->dev_ops.stop && genpd->dev_ops.start &&

> @@ -1409,23 +1413,28 @@ static void genpd_complete(struct device *dev)

>  static void genpd_switch_state(struct device *dev, bool suspend)

>  {

>         struct generic_pm_domain *genpd;

> +       struct generic_pm_domain_data *gpd_data;

>         bool use_lock;

>

>         genpd = dev_to_genpd_safe(dev);

>         if (!genpd)

>                 return;

>

> +       gpd_data = dev_gpd_data(dev);

> +

>         use_lock = genpd_is_irq_safe(genpd);

>

>         if (use_lock)

>                 genpd_lock(genpd);

>

>         if (suspend) {

> +               gpd_data->pm_pstate = genpd_drop_performance_state(dev);

>                 genpd->suspended_count++;

>                 genpd_sync_power_off(genpd, use_lock, 0);

>         } else {

>                 genpd_sync_power_on(genpd, use_lock, 0);

>                 genpd->suspended_count--;

> +               genpd_restore_performance_state(dev, gpd_data->pm_pstate);

>         }

>

>         if (use_lock)

> diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h

> index 21a0577305ef..f6e9dc28621c 100644

> --- a/include/linux/pm_domain.h

> +++ b/include/linux/pm_domain.h

> @@ -199,6 +199,7 @@ struct generic_pm_domain_data {

>         int cpu;

>         unsigned int performance_state;

>         unsigned int rpm_pstate;

> +       unsigned int pm_pstate;

>         ktime_t next_wakeup;

>         void *data;

>  };

> --

> 2.25.1

>
Mark Brown June 3, 2021, 11:15 a.m. | #2
On Thu, Jun 03, 2021 at 12:20:57PM +0200, Ulf Hansson wrote:
> On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:


> > Recent changes in genpd drops and restore performance state votes for

> > devices during runtime PM.


> After a second thought, it looks like we maybe should defer to apply

> this final patch of the series. At least until we figured out how to

> address the below issue:


> So, I noticed that we have things like "regulator-fixed-domain", that

> uses "required-opps" to enable/disable a regulator through the

> dev_pm_set_performance_state() interface. We likely don't want to drop

> the performance state internally in genpd when genpd_suspend_noirq()

> gets called, for the corresponding struct device for the regulator.


> I guess if genpd should drop performance states like $subject patch

> suggest, we need some kind of additional coordination, that allows a

> subsystem/driver to inform genpd when it should avoid it. Or something

> along those lines.


I'm not sure what you're looking for from me here - was there a concrete
question or somehing?
Ulf Hansson June 3, 2021, 1:48 p.m. | #3
On Thu, 3 Jun 2021 at 13:15, Mark Brown <broonie@kernel.org> wrote:
>

> On Thu, Jun 03, 2021 at 12:20:57PM +0200, Ulf Hansson wrote:

> > On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:

>

> > > Recent changes in genpd drops and restore performance state votes for

> > > devices during runtime PM.

>

> > After a second thought, it looks like we maybe should defer to apply

> > this final patch of the series. At least until we figured out how to

> > address the below issue:

>

> > So, I noticed that we have things like "regulator-fixed-domain", that

> > uses "required-opps" to enable/disable a regulator through the

> > dev_pm_set_performance_state() interface. We likely don't want to drop

> > the performance state internally in genpd when genpd_suspend_noirq()

> > gets called, for the corresponding struct device for the regulator.

>

> > I guess if genpd should drop performance states like $subject patch

> > suggest, we need some kind of additional coordination, that allows a

> > subsystem/driver to inform genpd when it should avoid it. Or something

> > along those lines.

>

> I'm not sure what you're looking for from me here - was there a concrete

> question or somehing?


Nope, not really, sorry if that was not clear.

I just wanted to loop you in, as to make sure that we don't change
something at the PM domain level, which may not fit well with the
regulator implementation.

Kind regards
Uffe
Stephan Gerhold June 8, 2021, 12:53 p.m. | #4
Hi,

On Thu, Jun 03, 2021 at 12:20:57PM +0200, Ulf Hansson wrote:
> + Mark Brown, Dmitry Baryshkov

> 

> On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:

> >

> > Recent changes in genpd drops and restore performance state votes for

> > devices during runtime PM.

> >

> > For the similar reasons, but to avoid the same kind of boilerplate code in

> > device PM callbacks for system sleep in subsystems/drivers, let's drop and

> > restore performance states votes in genpd for the attached devices during

> > system sleep.

> >

> > Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>

> 

> After a second thought, it looks like we maybe should defer to apply

> this final patch of the series. At least until we figured out how to

> address the below issue:

> 

> So, I noticed that we have things like "regulator-fixed-domain", that

> uses "required-opps" to enable/disable a regulator through the

> dev_pm_set_performance_state() interface.


Not directly related to your concern, but related to another discussion
we had recently: To me, this looks mostly like another solution for
voting for performance states without doing full DVFS, also known as
assigned-performance-states [1] or required-opps on devices [2]. :)

It's just wrapped in a regulator interface here. Actually, if we
implement [2], the regulator-fixed-domain should mostly just become some
sort of simple wrapper around runtime PM for the regulator device, since
the required-opp might be applied automatically then.

[1]: https://lore.kernel.org/linux-arm-msm/1622095949-2014-1-git-send-email-rnayak@codeaurora.org/
[2]: https://lore.kernel.org/linux-arm-msm/YLYV3ov%2FiBffZMg4@gerhold.net/

> We likely don't want to drop the performance state internally in genpd

> when genpd_suspend_noirq() gets called, for the corresponding struct

> device for the regulator.

> 


So your concern is that the performance state is dropped during suspend
even though the regulator core thinks the regulator stays enabled?

I played with regulator-fixed-domain a bit and I would say this is
already broken (unless you rely on one of the side effects I mentioned
in [3]). The power domain gets powered off entirely during system
suspend, and then the performance state won't have any effect either.

I guess we would need some way to say that this device should only be
managed through runtime PM and never automatically suspended during
system suspend?

Stephan

[3]: https://lore.kernel.org/linux-pm/YLkOAyydZMnxkEy+@gerhold.net/
Ulf Hansson June 8, 2021, 2:08 p.m. | #5
On Tue, 8 Jun 2021 at 14:53, Stephan Gerhold <stephan@gerhold.net> wrote:
>

> Hi,

>

> On Thu, Jun 03, 2021 at 12:20:57PM +0200, Ulf Hansson wrote:

> > + Mark Brown, Dmitry Baryshkov

> >

> > On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:

> > >

> > > Recent changes in genpd drops and restore performance state votes for

> > > devices during runtime PM.

> > >

> > > For the similar reasons, but to avoid the same kind of boilerplate code in

> > > device PM callbacks for system sleep in subsystems/drivers, let's drop and

> > > restore performance states votes in genpd for the attached devices during

> > > system sleep.

> > >

> > > Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>

> >

> > After a second thought, it looks like we maybe should defer to apply

> > this final patch of the series. At least until we figured out how to

> > address the below issue:

> >

> > So, I noticed that we have things like "regulator-fixed-domain", that

> > uses "required-opps" to enable/disable a regulator through the

> > dev_pm_set_performance_state() interface.

>

> Not directly related to your concern, but related to another discussion

> we had recently: To me, this looks mostly like another solution for

> voting for performance states without doing full DVFS, also known as

> assigned-performance-states [1] or required-opps on devices [2]. :)

>

> It's just wrapped in a regulator interface here. Actually, if we

> implement [2], the regulator-fixed-domain should mostly just become some

> sort of simple wrapper around runtime PM for the regulator device, since

> the required-opp might be applied automatically then.


Honestly, I am not sure about what the regulator-fixed-domain intends
to model, but I assume it's something that fits well to be modelled as
a plain regulator, to start with.

Perhaps Mark can chime in and spread some light over this?

>

> [1]: https://lore.kernel.org/linux-arm-msm/1622095949-2014-1-git-send-email-rnayak@codeaurora.org/

> [2]: https://lore.kernel.org/linux-arm-msm/YLYV3ov%2FiBffZMg4@gerhold.net/

>

> > We likely don't want to drop the performance state internally in genpd

> > when genpd_suspend_noirq() gets called, for the corresponding struct

> > device for the regulator.

> >

>

> So your concern is that the performance state is dropped during suspend

> even though the regulator core thinks the regulator stays enabled?


Yes.

>

> I played with regulator-fixed-domain a bit and I would say this is

> already broken (unless you rely on one of the side effects I mentioned

> in [3]). The power domain gets powered off entirely during system

> suspend, and then the performance state won't have any effect either.


Right, I get your point.

Although, this isn't a problem, because the on/off and performance
states are today considered as orthogonal in gendp. Well, at least
currently until/if we decide to change this.

>

> I guess we would need some way to say that this device should only be

> managed through runtime PM and never automatically suspended during

> system suspend?


Yes!

For the on/off state, genpd uses the system wakeup interface to
understand whether the device is used in a wakeup path, see the call
to device_wakeup_path() in genpd_finish_suspend().
If that's the case the PM domain stays powered on during system suspend.

Potentially we could use the same interface (or something similar) to
support these kinds of cases.

>

> Stephan

>

> [3]: https://lore.kernel.org/linux-pm/YLkOAyydZMnxkEy+@gerhold.net/


Kind regards
Uffe
Mark Brown June 8, 2021, 2:20 p.m. | #6
On Tue, Jun 08, 2021 at 04:08:55PM +0200, Ulf Hansson wrote:

> Honestly, I am not sure about what the regulator-fixed-domain intends

> to model, but I assume it's something that fits well to be modelled as

> a plain regulator, to start with.


> Perhaps Mark can chime in and spread some light over this?


IIRC it's for situations where there's a device that's normally built as
a separate chip that got built into a bigger SoC and wants to rear end
something onto a power domain, I guess especially if the power domain
doesn't cover the whole of a Linux device.
Ulf Hansson June 8, 2021, 2:39 p.m. | #7
On Tue, 8 Jun 2021 at 16:20, Mark Brown <broonie@kernel.org> wrote:
>

> On Tue, Jun 08, 2021 at 04:08:55PM +0200, Ulf Hansson wrote:

>

> > Honestly, I am not sure about what the regulator-fixed-domain intends

> > to model, but I assume it's something that fits well to be modelled as

> > a plain regulator, to start with.

>

> > Perhaps Mark can chime in and spread some light over this?

>

> IIRC it's for situations where there's a device that's normally built as

> a separate chip that got built into a bigger SoC and wants to rear end

> something onto a power domain, I guess especially if the power domain

> doesn't cover the whole of a Linux device.


Alright, thanks for explaining!

Certainly something that should not be mixed up with a regular power
rail for shared resources (aka power-domain). Maybe it's worth trying
to extend the description in the DT binding a bit, so it doesn't get
abused?

Kind regards
Uffe
Stephan Gerhold June 8, 2021, 3:37 p.m. | #8
On Tue, Jun 08, 2021 at 04:08:55PM +0200, Ulf Hansson wrote:
> On Tue, 8 Jun 2021 at 14:53, Stephan Gerhold <stephan@gerhold.net> wrote:

> >

> > Hi,

> >

> > On Thu, Jun 03, 2021 at 12:20:57PM +0200, Ulf Hansson wrote:

> > > + Mark Brown, Dmitry Baryshkov

> > >

> > > On Thu, 3 Jun 2021 at 11:34, Ulf Hansson <ulf.hansson@linaro.org> wrote:

> > > >

> > > > Recent changes in genpd drops and restore performance state votes for

> > > > devices during runtime PM.

> > > >

> > > > For the similar reasons, but to avoid the same kind of boilerplate code in

> > > > device PM callbacks for system sleep in subsystems/drivers, let's drop and

> > > > restore performance states votes in genpd for the attached devices during

> > > > system sleep.

> > > >

> > > > Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>

> > >

> > > After a second thought, it looks like we maybe should defer to apply

> > > this final patch of the series. At least until we figured out how to

> > > address the below issue:

> > >

> > > So, I noticed that we have things like "regulator-fixed-domain", that

> > > uses "required-opps" to enable/disable a regulator through the

> > > dev_pm_set_performance_state() interface.

> >

> > Not directly related to your concern, but related to another discussion

> > we had recently: To me, this looks mostly like another solution for

> > voting for performance states without doing full DVFS, also known as

> > assigned-performance-states [1] or required-opps on devices [2]. :)

> >

> > It's just wrapped in a regulator interface here. Actually, if we

> > implement [2], the regulator-fixed-domain should mostly just become some

> > sort of simple wrapper around runtime PM for the regulator device, since

> > the required-opp might be applied automatically then.

> 

> Honestly, I am not sure about what the regulator-fixed-domain intends

> to model, but I assume it's something that fits well to be modelled as

> a plain regulator, to start with.

> 

> Perhaps Mark can chime in and spread some light over this?

> 

> >

> > [1]: https://lore.kernel.org/linux-arm-msm/1622095949-2014-1-git-send-email-rnayak@codeaurora.org/

> > [2]: https://lore.kernel.org/linux-arm-msm/YLYV3ov%2FiBffZMg4@gerhold.net/

> >

> > > We likely don't want to drop the performance state internally in genpd

> > > when genpd_suspend_noirq() gets called, for the corresponding struct

> > > device for the regulator.

> > >

> >

> > So your concern is that the performance state is dropped during suspend

> > even though the regulator core thinks the regulator stays enabled?

> 

> Yes.

> 

> >

> > I played with regulator-fixed-domain a bit and I would say this is

> > already broken (unless you rely on one of the side effects I mentioned

> > in [3]). The power domain gets powered off entirely during system

> > suspend, and then the performance state won't have any effect either.

> 

> Right, I get your point.

> 

> Although, this isn't a problem, because the on/off and performance

> states are today considered as orthogonal in gendp. Well, at least

> currently until/if we decide to change this.

> 


And in practice applying your patch should not be a problem either. :)
The main user so far is arch/arm64/boot/dts/qcom/sm8250.dtsi with the
rpmhpd genpd provider. That one drops the performance states anyway
when the power domain gets powered off.

> >

> > I guess we would need some way to say that this device should only be

> > managed through runtime PM and never automatically suspended during

> > system suspend?

> 

> Yes!

> 

> For the on/off state, genpd uses the system wakeup interface to

> understand whether the device is used in a wakeup path, see the call

> to device_wakeup_path() in genpd_finish_suspend().

> If that's the case the PM domain stays powered on during system suspend.

> 

> Potentially we could use the same interface (or something similar) to

> support these kinds of cases.

> 


Hmm, I wonder if I need something similar for my CPU DVFS case as well.
In my testing back then the power domain for the CPU was powered off
during system suspend. This doesn't make sense, since it's the CPU that
is running all the system suspend code. :)

Actually in my case there is no need to do anything during system
suspend. I use a special MSM8916_VDDMX_AO power domain from the "rpmpd"
genpd provider where _AO means "active-only". I think it's some kind of
firmware mechanism to only apply the performance state vote if the CPU
is actually active and the cluster is not powered down.
(I hope that MSM8916 actually powers down the CPU cluster during
 system suspend/s2idle after all your PSCI cpuidle patches :))

So I think I will also need to opt-out from the effects of this patch.

Stephan

Patch

diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index e5d97174c254..a33e5b341f3f 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -1171,6 +1171,7 @@  static int genpd_prepare(struct device *dev)
  */
 static int genpd_finish_suspend(struct device *dev, bool poweroff)
 {
+	struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev);
 	struct generic_pm_domain *genpd;
 	int ret = 0;
 
@@ -1201,6 +1202,7 @@  static int genpd_finish_suspend(struct device *dev, bool poweroff)
 	}
 
 	genpd_lock(genpd);
+	gpd_data->pm_pstate = genpd_drop_performance_state(dev);
 	genpd->suspended_count++;
 	genpd_sync_power_off(genpd, true, 0);
 	genpd_unlock(genpd);
@@ -1245,6 +1247,7 @@  static int genpd_resume_noirq(struct device *dev)
 	genpd_lock(genpd);
 	genpd_sync_power_on(genpd, true, 0);
 	genpd->suspended_count--;
+	genpd_restore_performance_state(dev, dev_gpd_data(dev)->pm_pstate);
 	genpd_unlock(genpd);
 
 	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
@@ -1364,6 +1367,7 @@  static int genpd_restore_noirq(struct device *dev)
 	}
 
 	genpd_sync_power_on(genpd, true, 0);
+	genpd_restore_performance_state(dev, dev_gpd_data(dev)->pm_pstate);
 	genpd_unlock(genpd);
 
 	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
@@ -1409,23 +1413,28 @@  static void genpd_complete(struct device *dev)
 static void genpd_switch_state(struct device *dev, bool suspend)
 {
 	struct generic_pm_domain *genpd;
+	struct generic_pm_domain_data *gpd_data;
 	bool use_lock;
 
 	genpd = dev_to_genpd_safe(dev);
 	if (!genpd)
 		return;
 
+	gpd_data = dev_gpd_data(dev);
+
 	use_lock = genpd_is_irq_safe(genpd);
 
 	if (use_lock)
 		genpd_lock(genpd);
 
 	if (suspend) {
+		gpd_data->pm_pstate = genpd_drop_performance_state(dev);
 		genpd->suspended_count++;
 		genpd_sync_power_off(genpd, use_lock, 0);
 	} else {
 		genpd_sync_power_on(genpd, use_lock, 0);
 		genpd->suspended_count--;
+		genpd_restore_performance_state(dev, gpd_data->pm_pstate);
 	}
 
 	if (use_lock)
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index 21a0577305ef..f6e9dc28621c 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -199,6 +199,7 @@  struct generic_pm_domain_data {
 	int cpu;
 	unsigned int performance_state;
 	unsigned int rpm_pstate;
+	unsigned int pm_pstate;
 	ktime_t	next_wakeup;
 	void *data;
 };