diff mbox series

[RFC,v3,21/21] cpumask: Add enabled cpumask for present CPUs that can be brought online

Message ID E1rDOhX-00Dvlg-Ci@rmk-PC.armlinux.org.uk
State Superseded
Headers show
Series ACPI/arm64: add support for virtual cpu hotplug | expand

Commit Message

Russell King (Oracle) Dec. 13, 2023, 12:50 p.m. UTC
From: James Morse <james.morse@arm.com>

The 'offline' file in sysfs shows all offline CPUs, including those
that aren't present. User-space is expected to remove not-present CPUs
from this list to learn which CPUs could be brought online.

CPUs can be present but not-enabled. These CPUs can't be brought online
until the firmware policy changes, which comes with an ACPI notification
that will register the CPUs.

With only the offline and present files, user-space is unable to
determine which CPUs it can try to bring online. Add a new CPU mask
that shows this based on all the registered CPUs.

Signed-off-by: James Morse <james.morse@arm.com>
Tested-by: Miguel Luis <miguel.luis@oracle.com>
Tested-by: Vishnu Pajjuri <vishnu@os.amperecomputing.com>
Tested-by: Jianyong Wu <jianyong.wu@arm.com>
---
Outstanding comment:
 https://lore.kernel.org/r/20230914175443.000038f6@Huawei.com
---
 drivers/base/cpu.c      | 10 ++++++++++
 include/linux/cpumask.h | 25 +++++++++++++++++++++++++
 kernel/cpu.c            |  3 +++
 3 files changed, 38 insertions(+)

Comments

Jonathan Cameron Dec. 15, 2023, 5:18 p.m. UTC | #1
On Wed, 13 Dec 2023 12:50:59 +0000
Russell King (Oracle) <rmk+kernel@armlinux.org.uk> wrote:

> From: James Morse <james.morse@arm.com>
> 
> The 'offline' file in sysfs shows all offline CPUs, including those
> that aren't present. User-space is expected to remove not-present CPUs
> from this list to learn which CPUs could be brought online.
> 
> CPUs can be present but not-enabled. These CPUs can't be brought online
> until the firmware policy changes, which comes with an ACPI notification
> that will register the CPUs.
> 
> With only the offline and present files, user-space is unable to
> determine which CPUs it can try to bring online. Add a new CPU mask
> that shows this based on all the registered CPUs.
> 
> Signed-off-by: James Morse <james.morse@arm.com>
> Tested-by: Miguel Luis <miguel.luis@oracle.com>
> Tested-by: Vishnu Pajjuri <vishnu@os.amperecomputing.com>
> Tested-by: Jianyong Wu <jianyong.wu@arm.com>
> ---

Needs docs
Documentation/ABI/testing/sysfs-devices-system-cpu
seems to have the rest of the similar entries.

> Outstanding comment:
>  https://lore.kernel.org/r/20230914175443.000038f6@Huawei.com
Very fussy reviewer. I'd ignore him on this :)

Code is fine.

Thanks for taking this forwards.  Maybe the end of this saga is
close!

Jonathan

> ---
>  drivers/base/cpu.c      | 10 ++++++++++
>  include/linux/cpumask.h | 25 +++++++++++++++++++++++++
>  kernel/cpu.c            |  3 +++
>  3 files changed, 38 insertions(+)
> 
> diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
> index 13d052bf13f4..a6e96a0a92b7 100644
> --- a/drivers/base/cpu.c
> +++ b/drivers/base/cpu.c
> @@ -95,6 +95,7 @@ void unregister_cpu(struct cpu *cpu)
>  {
>  	int logical_cpu = cpu->dev.id;
>  
> +	set_cpu_enabled(logical_cpu, false);
>  	unregister_cpu_under_node(logical_cpu, cpu_to_node(logical_cpu));
>  
>  	device_unregister(&cpu->dev);
> @@ -273,6 +274,13 @@ static ssize_t print_cpus_offline(struct device *dev,
>  }
>  static DEVICE_ATTR(offline, 0444, print_cpus_offline, NULL);
>  
> +static ssize_t print_cpus_enabled(struct device *dev,
> +				  struct device_attribute *attr, char *buf)
> +{
> +	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_enabled_mask));
> +}
> +static DEVICE_ATTR(enabled, 0444, print_cpus_enabled, NULL);
> +
>  static ssize_t print_cpus_isolated(struct device *dev,
>  				  struct device_attribute *attr, char *buf)
>  {
> @@ -413,6 +421,7 @@ int register_cpu(struct cpu *cpu, int num)
>  	register_cpu_under_node(num, cpu_to_node(num));
>  	dev_pm_qos_expose_latency_limit(&cpu->dev,
>  					PM_QOS_RESUME_LATENCY_NO_CONSTRAINT);
> +	set_cpu_enabled(num, true);
>  
>  	return 0;
>  }
> @@ -494,6 +503,7 @@ static struct attribute *cpu_root_attrs[] = {
>  	&cpu_attrs[2].attr.attr,
>  	&dev_attr_kernel_max.attr,
>  	&dev_attr_offline.attr,
> +	&dev_attr_enabled.attr,
>  	&dev_attr_isolated.attr,
>  #ifdef CONFIG_NO_HZ_FULL
>  	&dev_attr_nohz_full.attr,
> diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
> index cfb545841a2c..cc72a0887f04 100644
> --- a/include/linux/cpumask.h
> +++ b/include/linux/cpumask.h
> @@ -92,6 +92,7 @@ static inline void set_nr_cpu_ids(unsigned int nr)
>   *
>   *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
>   *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
> + *     cpu_enabled_mask  - has bit 'cpu' set iff cpu can be brought online
>   *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
>   *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
>   *
> @@ -124,11 +125,13 @@ static inline void set_nr_cpu_ids(unsigned int nr)
>  
>  extern struct cpumask __cpu_possible_mask;
>  extern struct cpumask __cpu_online_mask;
> +extern struct cpumask __cpu_enabled_mask;
>  extern struct cpumask __cpu_present_mask;
>  extern struct cpumask __cpu_active_mask;
>  extern struct cpumask __cpu_dying_mask;
>  #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
>  #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
> +#define cpu_enabled_mask   ((const struct cpumask *)&__cpu_enabled_mask)
>  #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
>  #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
>  #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
> @@ -993,6 +996,7 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
>  #else
>  #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
>  #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
> +#define for_each_enabled_cpu(cpu)   for_each_cpu((cpu), cpu_enabled_mask)
>  #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
>  #endif
>  
> @@ -1015,6 +1019,15 @@ set_cpu_possible(unsigned int cpu, bool possible)
>  		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
>  }
>  
> +static inline void
> +set_cpu_enabled(unsigned int cpu, bool can_be_onlined)
> +{
> +	if (can_be_onlined)
> +		cpumask_set_cpu(cpu, &__cpu_enabled_mask);
> +	else
> +		cpumask_clear_cpu(cpu, &__cpu_enabled_mask);
> +}
> +
>  static inline void
>  set_cpu_present(unsigned int cpu, bool present)
>  {
> @@ -1096,6 +1109,7 @@ static __always_inline unsigned int num_online_cpus(void)
>  	return raw_atomic_read(&__num_online_cpus);
>  }
>  #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
> +#define num_enabled_cpus()	cpumask_weight(cpu_enabled_mask)
>  #define num_present_cpus()	cpumask_weight(cpu_present_mask)
>  #define num_active_cpus()	cpumask_weight(cpu_active_mask)
>  
> @@ -1104,6 +1118,11 @@ static inline bool cpu_online(unsigned int cpu)
>  	return cpumask_test_cpu(cpu, cpu_online_mask);
>  }
>  
> +static inline bool cpu_enabled(unsigned int cpu)
> +{
> +	return cpumask_test_cpu(cpu, cpu_enabled_mask);
> +}
> +
>  static inline bool cpu_possible(unsigned int cpu)
>  {
>  	return cpumask_test_cpu(cpu, cpu_possible_mask);
> @@ -1128,6 +1147,7 @@ static inline bool cpu_dying(unsigned int cpu)
>  
>  #define num_online_cpus()	1U
>  #define num_possible_cpus()	1U
> +#define num_enabled_cpus()	1U
>  #define num_present_cpus()	1U
>  #define num_active_cpus()	1U
>  
> @@ -1141,6 +1161,11 @@ static inline bool cpu_possible(unsigned int cpu)
>  	return cpu == 0;
>  }
>  
> +static inline bool cpu_enabled(unsigned int cpu)
> +{
> +	return cpu == 0;
> +}
> +
>  static inline bool cpu_present(unsigned int cpu)
>  {
>  	return cpu == 0;
> diff --git a/kernel/cpu.c b/kernel/cpu.c
> index a86972a91991..fe0a5189f8ae 100644
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -3122,6 +3122,9 @@ EXPORT_SYMBOL(__cpu_possible_mask);
>  struct cpumask __cpu_online_mask __read_mostly;
>  EXPORT_SYMBOL(__cpu_online_mask);
>  
> +struct cpumask __cpu_enabled_mask __read_mostly;
> +EXPORT_SYMBOL(__cpu_enabled_mask);
> +
>  struct cpumask __cpu_present_mask __read_mostly;
>  EXPORT_SYMBOL(__cpu_present_mask);
>
Thomas Gleixner Dec. 15, 2023, 7:40 p.m. UTC | #2
On Wed, Dec 13 2023 at 12:50, Russell King (Oracle) wrote:
> From: James Morse <james.morse@arm.com>
>
> The 'offline' file in sysfs shows all offline CPUs, including those
> that aren't present. User-space is expected to remove not-present CPUs
> from this list to learn which CPUs could be brought online.
>
> CPUs can be present but not-enabled. These CPUs can't be brought online
> until the firmware policy changes, which comes with an ACPI notification
> that will register the CPUs.
>
> With only the offline and present files, user-space is unable to
> determine which CPUs it can try to bring online. Add a new CPU mask
> that shows this based on all the registered CPUs.
>
> Signed-off-by: James Morse <james.morse@arm.com>
> Tested-by: Miguel Luis <miguel.luis@oracle.com>
> Tested-by: Vishnu Pajjuri <vishnu@os.amperecomputing.com>
> Tested-by: Jianyong Wu <jianyong.wu@arm.com>

Acked-by: Thomas Gleixner <tglx@linutronix.de>
Russell King (Oracle) Dec. 18, 2023, 12:14 p.m. UTC | #3
On Fri, Dec 15, 2023 at 05:18:31PM +0000, Jonathan Cameron wrote:
> On Wed, 13 Dec 2023 12:50:59 +0000
> Russell King (Oracle) <rmk+kernel@armlinux.org.uk> wrote:
> 
> > From: James Morse <james.morse@arm.com>
> > 
> > The 'offline' file in sysfs shows all offline CPUs, including those
> > that aren't present. User-space is expected to remove not-present CPUs
> > from this list to learn which CPUs could be brought online.
> > 
> > CPUs can be present but not-enabled. These CPUs can't be brought online
> > until the firmware policy changes, which comes with an ACPI notification
> > that will register the CPUs.
> > 
> > With only the offline and present files, user-space is unable to
> > determine which CPUs it can try to bring online. Add a new CPU mask
> > that shows this based on all the registered CPUs.
> > 
> > Signed-off-by: James Morse <james.morse@arm.com>
> > Tested-by: Miguel Luis <miguel.luis@oracle.com>
> > Tested-by: Vishnu Pajjuri <vishnu@os.amperecomputing.com>
> > Tested-by: Jianyong Wu <jianyong.wu@arm.com>
> > ---
> 
> Needs docs
> Documentation/ABI/testing/sysfs-devices-system-cpu
> seems to have the rest of the similar entries.

Any ideas what I put in there as "Date" ? It seems to me that we have
little idea when this might be merged.. I could use the date of the
commit (Nov 2022).
Jonathan Cameron Jan. 2, 2024, 3:19 p.m. UTC | #4
On Mon, 18 Dec 2023 12:14:14 +0000
"Russell King (Oracle)" <linux@armlinux.org.uk> wrote:

> On Fri, Dec 15, 2023 at 05:18:31PM +0000, Jonathan Cameron wrote:
> > On Wed, 13 Dec 2023 12:50:59 +0000
> > Russell King (Oracle) <rmk+kernel@armlinux.org.uk> wrote:
> >   
> > > From: James Morse <james.morse@arm.com>
> > > 
> > > The 'offline' file in sysfs shows all offline CPUs, including those
> > > that aren't present. User-space is expected to remove not-present CPUs
> > > from this list to learn which CPUs could be brought online.
> > > 
> > > CPUs can be present but not-enabled. These CPUs can't be brought online
> > > until the firmware policy changes, which comes with an ACPI notification
> > > that will register the CPUs.
> > > 
> > > With only the offline and present files, user-space is unable to
> > > determine which CPUs it can try to bring online. Add a new CPU mask
> > > that shows this based on all the registered CPUs.
> > > 
> > > Signed-off-by: James Morse <james.morse@arm.com>
> > > Tested-by: Miguel Luis <miguel.luis@oracle.com>
> > > Tested-by: Vishnu Pajjuri <vishnu@os.amperecomputing.com>
> > > Tested-by: Jianyong Wu <jianyong.wu@arm.com>
> > > ---  
> > 
> > Needs docs
> > Documentation/ABI/testing/sysfs-devices-system-cpu
> > seems to have the rest of the similar entries.  
> 
> Any ideas what I put in there as "Date" ? It seems to me that we have
> little idea when this might be merged.. I could use the date of the
> commit (Nov 2022).
> 

That's always a guess at best.  Hopefully whoever picks this up
fixes the date up or asks for a new version with it fixed just before
they do.

J
diff mbox series

Patch

diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 13d052bf13f4..a6e96a0a92b7 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -95,6 +95,7 @@  void unregister_cpu(struct cpu *cpu)
 {
 	int logical_cpu = cpu->dev.id;
 
+	set_cpu_enabled(logical_cpu, false);
 	unregister_cpu_under_node(logical_cpu, cpu_to_node(logical_cpu));
 
 	device_unregister(&cpu->dev);
@@ -273,6 +274,13 @@  static ssize_t print_cpus_offline(struct device *dev,
 }
 static DEVICE_ATTR(offline, 0444, print_cpus_offline, NULL);
 
+static ssize_t print_cpus_enabled(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_enabled_mask));
+}
+static DEVICE_ATTR(enabled, 0444, print_cpus_enabled, NULL);
+
 static ssize_t print_cpus_isolated(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
@@ -413,6 +421,7 @@  int register_cpu(struct cpu *cpu, int num)
 	register_cpu_under_node(num, cpu_to_node(num));
 	dev_pm_qos_expose_latency_limit(&cpu->dev,
 					PM_QOS_RESUME_LATENCY_NO_CONSTRAINT);
+	set_cpu_enabled(num, true);
 
 	return 0;
 }
@@ -494,6 +503,7 @@  static struct attribute *cpu_root_attrs[] = {
 	&cpu_attrs[2].attr.attr,
 	&dev_attr_kernel_max.attr,
 	&dev_attr_offline.attr,
+	&dev_attr_enabled.attr,
 	&dev_attr_isolated.attr,
 #ifdef CONFIG_NO_HZ_FULL
 	&dev_attr_nohz_full.attr,
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index cfb545841a2c..cc72a0887f04 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -92,6 +92,7 @@  static inline void set_nr_cpu_ids(unsigned int nr)
  *
  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
+ *     cpu_enabled_mask  - has bit 'cpu' set iff cpu can be brought online
  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
  *
@@ -124,11 +125,13 @@  static inline void set_nr_cpu_ids(unsigned int nr)
 
 extern struct cpumask __cpu_possible_mask;
 extern struct cpumask __cpu_online_mask;
+extern struct cpumask __cpu_enabled_mask;
 extern struct cpumask __cpu_present_mask;
 extern struct cpumask __cpu_active_mask;
 extern struct cpumask __cpu_dying_mask;
 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
 #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
+#define cpu_enabled_mask   ((const struct cpumask *)&__cpu_enabled_mask)
 #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
 #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
 #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
@@ -993,6 +996,7 @@  extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
 #else
 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
+#define for_each_enabled_cpu(cpu)   for_each_cpu((cpu), cpu_enabled_mask)
 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
 #endif
 
@@ -1015,6 +1019,15 @@  set_cpu_possible(unsigned int cpu, bool possible)
 		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
 }
 
+static inline void
+set_cpu_enabled(unsigned int cpu, bool can_be_onlined)
+{
+	if (can_be_onlined)
+		cpumask_set_cpu(cpu, &__cpu_enabled_mask);
+	else
+		cpumask_clear_cpu(cpu, &__cpu_enabled_mask);
+}
+
 static inline void
 set_cpu_present(unsigned int cpu, bool present)
 {
@@ -1096,6 +1109,7 @@  static __always_inline unsigned int num_online_cpus(void)
 	return raw_atomic_read(&__num_online_cpus);
 }
 #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
+#define num_enabled_cpus()	cpumask_weight(cpu_enabled_mask)
 #define num_present_cpus()	cpumask_weight(cpu_present_mask)
 #define num_active_cpus()	cpumask_weight(cpu_active_mask)
 
@@ -1104,6 +1118,11 @@  static inline bool cpu_online(unsigned int cpu)
 	return cpumask_test_cpu(cpu, cpu_online_mask);
 }
 
+static inline bool cpu_enabled(unsigned int cpu)
+{
+	return cpumask_test_cpu(cpu, cpu_enabled_mask);
+}
+
 static inline bool cpu_possible(unsigned int cpu)
 {
 	return cpumask_test_cpu(cpu, cpu_possible_mask);
@@ -1128,6 +1147,7 @@  static inline bool cpu_dying(unsigned int cpu)
 
 #define num_online_cpus()	1U
 #define num_possible_cpus()	1U
+#define num_enabled_cpus()	1U
 #define num_present_cpus()	1U
 #define num_active_cpus()	1U
 
@@ -1141,6 +1161,11 @@  static inline bool cpu_possible(unsigned int cpu)
 	return cpu == 0;
 }
 
+static inline bool cpu_enabled(unsigned int cpu)
+{
+	return cpu == 0;
+}
+
 static inline bool cpu_present(unsigned int cpu)
 {
 	return cpu == 0;
diff --git a/kernel/cpu.c b/kernel/cpu.c
index a86972a91991..fe0a5189f8ae 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3122,6 +3122,9 @@  EXPORT_SYMBOL(__cpu_possible_mask);
 struct cpumask __cpu_online_mask __read_mostly;
 EXPORT_SYMBOL(__cpu_online_mask);
 
+struct cpumask __cpu_enabled_mask __read_mostly;
+EXPORT_SYMBOL(__cpu_enabled_mask);
+
 struct cpumask __cpu_present_mask __read_mostly;
 EXPORT_SYMBOL(__cpu_present_mask);