[Xen-devel] xen: arm: increase priority of SGIs used as IPIs

Message ID 1390927878-7048-1-git-send-email-ian.campbell@citrix.com
State New
Headers show

Commit Message

Ian Campbell Jan. 28, 2014, 4:51 p.m.
Code such as on_selected_cpus expects/requires that an IPI can preempt a
processor which is just handling a normal interrupt. Lacking this property can
result in a deadlock between two CPUs trying to IPI each other from interrupt
context.

For the time being there is only two priorities, IRQ and IPI, although it is
also conceivable that in the future some IPIs might be higher priority than
others. This could be used to implement a better BUG() than we have now, but I
haven't tackled that yet.

Tested with a debug patch which sends a local IPI from a keyhandler, which is
run in serial interrupt context.

This should also fix the issue reported by Oleksandr in "xen/arm:
maintenance_interrupt SMP fix" without resorting to trylock.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Cc: Oleksandr Tyshchenko <oleksandr.tyshchenko@globallogic.com>
---
I think this is probably 4.5 material at this point.

Tested with "HACK: dump pcpu state keyhandler" which I'll post for
completeness. It gives:
(XEN) Xen call trace:
(XEN)    [<0000000000212048>] dump_pcpus+0x28/0x2c (PC)
(XEN)    [<000000000021256c>] handle_keypress+0x70/0xb0 (LR)
(XEN)    [<000000000023ed00>] __serial_rx+0x20/0x6c
(XEN)    [<000000000023f8ac>] serial_rx+0xb4/0xc4
(XEN)    [<00000000002409ec>] serial_rx_interrupt+0xb0/0xd4
(XEN)    [<00000000002404b4>] ns16550_interrupt+0x6c/0x90
(XEN)    [<0000000000245fc0>] do_IRQ+0x144/0x1b4
(XEN)    [<0000000000245a28>] gic_interrupt+0x60/0xf8
(XEN)    [<000000000024be64>] do_trap_irq+0x10/0x18
(XEN)    [<000000000024e240>] hyp_irq+0x5c/0x60
(XEN)    [<0000000000249324>] init_done+0x10/0x18
(XEN)    [<0000000000000080>] 0000000000000080
---
 xen/arch/arm/gic.c        |   19 +++++++++++++------
 xen/arch/arm/time.c       |    6 +++---
 xen/include/asm-arm/gic.h |   22 ++++++++++++++++++++++
 3 files changed, 38 insertions(+), 9 deletions(-)

Comments

Stefano Stabellini Jan. 28, 2014, 5:50 p.m. | #1
On Tue, 28 Jan 2014, Ian Campbell wrote:
> Code such as on_selected_cpus expects/requires that an IPI can preempt a
> processor which is just handling a normal interrupt. Lacking this property can
> result in a deadlock between two CPUs trying to IPI each other from interrupt
> context.
> 
> For the time being there is only two priorities, IRQ and IPI, although it is
> also conceivable that in the future some IPIs might be higher priority than
> others. This could be used to implement a better BUG() than we have now, but I
> haven't tackled that yet.
> 
> Tested with a debug patch which sends a local IPI from a keyhandler, which is
> run in serial interrupt context.
> 
> This should also fix the issue reported by Oleksandr in "xen/arm:
> maintenance_interrupt SMP fix" without resorting to trylock.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> Cc: Oleksandr Tyshchenko <oleksandr.tyshchenko@globallogic.com>

It looks simple enough.
Oleksandr, I would appreciate if could test the patch and tell us if it
is working well for you.


> I think this is probably 4.5 material at this point.
> 
> Tested with "HACK: dump pcpu state keyhandler" which I'll post for
> completeness. It gives:
> (XEN) Xen call trace:
> (XEN)    [<0000000000212048>] dump_pcpus+0x28/0x2c (PC)
> (XEN)    [<000000000021256c>] handle_keypress+0x70/0xb0 (LR)
> (XEN)    [<000000000023ed00>] __serial_rx+0x20/0x6c
> (XEN)    [<000000000023f8ac>] serial_rx+0xb4/0xc4
> (XEN)    [<00000000002409ec>] serial_rx_interrupt+0xb0/0xd4
> (XEN)    [<00000000002404b4>] ns16550_interrupt+0x6c/0x90
> (XEN)    [<0000000000245fc0>] do_IRQ+0x144/0x1b4
> (XEN)    [<0000000000245a28>] gic_interrupt+0x60/0xf8
> (XEN)    [<000000000024be64>] do_trap_irq+0x10/0x18
> (XEN)    [<000000000024e240>] hyp_irq+0x5c/0x60
> (XEN)    [<0000000000249324>] init_done+0x10/0x18
> (XEN)    [<0000000000000080>] 0000000000000080
> ---
>  xen/arch/arm/gic.c        |   19 +++++++++++++------
>  xen/arch/arm/time.c       |    6 +++---
>  xen/include/asm-arm/gic.h |   22 ++++++++++++++++++++++
>  3 files changed, 38 insertions(+), 9 deletions(-)
> 
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index dcf9cd4..ee37019 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -319,7 +319,8 @@ static void __init gic_dist_init(void)
>  
>      /* Default priority for global interrupts */
>      for ( i = 32; i < gic.lines; i += 4 )
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Disable all global interrupts */
>      for ( i = 32; i < gic.lines; i += 32 )
> @@ -341,8 +342,12 @@ static void __cpuinit gic_cpu_init(void)
>      GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
>      GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
>      /* Set PPI and SGI priorities */
> -    for (i = 0; i < 32; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +    for (i = 0; i < 16; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> +    for (i = 16; i < 32; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Local settings: interface controller */
>      GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
> @@ -538,7 +543,8 @@ void gic_disable_cpu(void)
>  void gic_route_ppis(void)
>  {
>      /* GIC maintenance */
> -    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()), 0xa0);
> +    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
> +                     GIC_PRI_IRQ);
>      /* Route timer interrupt */
>      route_timer_interrupt();
>  }
> @@ -553,7 +559,8 @@ void gic_route_spis(void)
>          if ( (irq = serial_dt_irq(seridx)) == NULL )
>              continue;
>  
> -        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()), 0xa0);
> +        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()),
> +                         GIC_PRI_IRQ);
>      }
>  }
>  
> @@ -777,7 +784,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>      level = dt_irq_is_level_triggered(irq);
>  
>      gic_set_irq_properties(irq->irq, level, cpumask_of(smp_processor_id()),
> -                           0xa0);
> +                           GIC_PRI_IRQ);
>  
>      retval = __setup_irq(desc, irq->irq, action);
>      if (retval) {
> diff --git a/xen/arch/arm/time.c b/xen/arch/arm/time.c
> index 81e3e28..68b939d 100644
> --- a/xen/arch/arm/time.c
> +++ b/xen/arch/arm/time.c
> @@ -222,11 +222,11 @@ static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
>  void __cpuinit route_timer_interrupt(void)
>  {
>      gic_route_dt_irq(&timer_irq[TIMER_PHYS_NONSECURE_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_HYP_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_VIRT_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>  }
>  
>  /* Set up the timer interrupt on this CPU */
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 9c6f9bb..25b2b24 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -129,6 +129,28 @@
>  #define GICH_LR_CPUID_SHIFT     9
>  #define GICH_VTR_NRLRGS         0x3f
>  
> +/*
> + * The minimum GICC_BPR is required to be in the range 0-3. We set
> + * GICC_BPR to 0 but we must expect that it might be 3. This means we
> + * can rely on premption between the following ranges:
> + * 0xf0..0xff
> + * 0xe0..0xdf
> + * 0xc0..0xcf
> + * 0xb0..0xbf
> + * 0xa0..0xaf
> + * 0x90..0x9f
> + * 0x80..0x8f
> + *
> + * Priorities within a range will not preempt each other.
> + *
> + * A GIC must support a mimimum of 16 priority levels.
> + */
> +#define GIC_PRI_LOWEST     0xf0
> +#define GIC_PRI_IRQ        0xa0
> +#define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
> +#define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
> +
> +
>  #ifndef __ASSEMBLY__
>  #include <xen/device_tree.h>
>  
> -- 
> 1.7.10.4
>
Ian Campbell March 13, 2014, 12:26 p.m. | #2
Ping?

I think this patch was shown to be an improvement but it lacks an actual
ack.

On Tue, 2014-01-28 at 16:51 +0000, Ian Campbell wrote:
> Code such as on_selected_cpus expects/requires that an IPI can preempt a
> processor which is just handling a normal interrupt. Lacking this property can
> result in a deadlock between two CPUs trying to IPI each other from interrupt
> context.
> 
> For the time being there is only two priorities, IRQ and IPI, although it is
> also conceivable that in the future some IPIs might be higher priority than
> others. This could be used to implement a better BUG() than we have now, but I
> haven't tackled that yet.
> 
> Tested with a debug patch which sends a local IPI from a keyhandler, which is
> run in serial interrupt context.
> 
> This should also fix the issue reported by Oleksandr in "xen/arm:
> maintenance_interrupt SMP fix" without resorting to trylock.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> Cc: Oleksandr Tyshchenko <oleksandr.tyshchenko@globallogic.com>
> ---
> I think this is probably 4.5 material at this point.
> 
> Tested with "HACK: dump pcpu state keyhandler" which I'll post for
> completeness. It gives:
> (XEN) Xen call trace:
> (XEN)    [<0000000000212048>] dump_pcpus+0x28/0x2c (PC)
> (XEN)    [<000000000021256c>] handle_keypress+0x70/0xb0 (LR)
> (XEN)    [<000000000023ed00>] __serial_rx+0x20/0x6c
> (XEN)    [<000000000023f8ac>] serial_rx+0xb4/0xc4
> (XEN)    [<00000000002409ec>] serial_rx_interrupt+0xb0/0xd4
> (XEN)    [<00000000002404b4>] ns16550_interrupt+0x6c/0x90
> (XEN)    [<0000000000245fc0>] do_IRQ+0x144/0x1b4
> (XEN)    [<0000000000245a28>] gic_interrupt+0x60/0xf8
> (XEN)    [<000000000024be64>] do_trap_irq+0x10/0x18
> (XEN)    [<000000000024e240>] hyp_irq+0x5c/0x60
> (XEN)    [<0000000000249324>] init_done+0x10/0x18
> (XEN)    [<0000000000000080>] 0000000000000080
> ---
>  xen/arch/arm/gic.c        |   19 +++++++++++++------
>  xen/arch/arm/time.c       |    6 +++---
>  xen/include/asm-arm/gic.h |   22 ++++++++++++++++++++++
>  3 files changed, 38 insertions(+), 9 deletions(-)
> 
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index dcf9cd4..ee37019 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -319,7 +319,8 @@ static void __init gic_dist_init(void)
>  
>      /* Default priority for global interrupts */
>      for ( i = 32; i < gic.lines; i += 4 )
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Disable all global interrupts */
>      for ( i = 32; i < gic.lines; i += 32 )
> @@ -341,8 +342,12 @@ static void __cpuinit gic_cpu_init(void)
>      GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
>      GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
>      /* Set PPI and SGI priorities */
> -    for (i = 0; i < 32; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +    for (i = 0; i < 16; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> +    for (i = 16; i < 32; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Local settings: interface controller */
>      GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
> @@ -538,7 +543,8 @@ void gic_disable_cpu(void)
>  void gic_route_ppis(void)
>  {
>      /* GIC maintenance */
> -    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()), 0xa0);
> +    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
> +                     GIC_PRI_IRQ);
>      /* Route timer interrupt */
>      route_timer_interrupt();
>  }
> @@ -553,7 +559,8 @@ void gic_route_spis(void)
>          if ( (irq = serial_dt_irq(seridx)) == NULL )
>              continue;
>  
> -        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()), 0xa0);
> +        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()),
> +                         GIC_PRI_IRQ);
>      }
>  }
>  
> @@ -777,7 +784,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>      level = dt_irq_is_level_triggered(irq);
>  
>      gic_set_irq_properties(irq->irq, level, cpumask_of(smp_processor_id()),
> -                           0xa0);
> +                           GIC_PRI_IRQ);
>  
>      retval = __setup_irq(desc, irq->irq, action);
>      if (retval) {
> diff --git a/xen/arch/arm/time.c b/xen/arch/arm/time.c
> index 81e3e28..68b939d 100644
> --- a/xen/arch/arm/time.c
> +++ b/xen/arch/arm/time.c
> @@ -222,11 +222,11 @@ static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
>  void __cpuinit route_timer_interrupt(void)
>  {
>      gic_route_dt_irq(&timer_irq[TIMER_PHYS_NONSECURE_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_HYP_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_VIRT_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>  }
>  
>  /* Set up the timer interrupt on this CPU */
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 9c6f9bb..25b2b24 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -129,6 +129,28 @@
>  #define GICH_LR_CPUID_SHIFT     9
>  #define GICH_VTR_NRLRGS         0x3f
>  
> +/*
> + * The minimum GICC_BPR is required to be in the range 0-3. We set
> + * GICC_BPR to 0 but we must expect that it might be 3. This means we
> + * can rely on premption between the following ranges:
> + * 0xf0..0xff
> + * 0xe0..0xdf
> + * 0xc0..0xcf
> + * 0xb0..0xbf
> + * 0xa0..0xaf
> + * 0x90..0x9f
> + * 0x80..0x8f
> + *
> + * Priorities within a range will not preempt each other.
> + *
> + * A GIC must support a mimimum of 16 priority levels.
> + */
> +#define GIC_PRI_LOWEST     0xf0
> +#define GIC_PRI_IRQ        0xa0
> +#define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
> +#define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
> +
> +
>  #ifndef __ASSEMBLY__
>  #include <xen/device_tree.h>
>
Julien Grall March 13, 2014, 12:33 p.m. | #3
Hi Ian,

On 01/28/2014 04:51 PM, Ian Campbell wrote:
>      /* Set PPI and SGI priorities */
> -    for (i = 0; i < 32; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +    for (i = 0; i < 16; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> +    for (i = 16; i < 32; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;

I'm wondering if it's necessary to set the priority for PPIs. It will be
overridden later when the interrupt will be setup. Until that time, the
interrupt is not enabled.

Regards,
Ian Campbell March 13, 2014, 12:45 p.m. | #4
On Thu, 2014-03-13 at 12:33 +0000, Julien Grall wrote:
> Hi Ian,
> 
> On 01/28/2014 04:51 PM, Ian Campbell wrote:
> >      /* Set PPI and SGI priorities */
> > -    for (i = 0; i < 32; i += 4)
> > -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> > +    for (i = 0; i < 16; i += 4)
> > +        GICD[GICD_IPRIORITYR + i / 4] =
> > +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> > +    for (i = 16; i < 32; i += 4)
> > +        GICD[GICD_IPRIORITYR + i / 4] =
> > +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> 
> I'm wondering if it's necessary to set the priority for PPIs. It will be
> overridden later when the interrupt will be setup. Until that time, the
> interrupt is not enabled.

Strictly speaking probably not, but there isn't much harm in setting a
sane default I suppose.

Ian.
Julien Grall March 13, 2014, 12:56 p.m. | #5
On 03/13/2014 12:45 PM, Ian Campbell wrote:
> On Thu, 2014-03-13 at 12:33 +0000, Julien Grall wrote:
>> Hi Ian,
>>
>> On 01/28/2014 04:51 PM, Ian Campbell wrote:
>>>      /* Set PPI and SGI priorities */
>>> -    for (i = 0; i < 32; i += 4)
>>> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
>>> +    for (i = 0; i < 16; i += 4)
>>> +        GICD[GICD_IPRIORITYR + i / 4] =
>>> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
>>> +    for (i = 16; i < 32; i += 4)
>>> +        GICD[GICD_IPRIORITYR + i / 4] =
>>> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>>
>> I'm wondering if it's necessary to set the priority for PPIs. It will be
>> overridden later when the interrupt will be setup. Until that time, the
>> interrupt is not enabled.
> 
> Strictly speaking probably not, but there isn't much harm in setting a
> sane default I suppose.

Right, I didn't see that we also do that for SPIs.

I have a minor change to request, can you divide the comment "Set PPI
and SGI priorities" in 2 to reflect was does each loop?

Except that:

Acked-by: Julien Grall <julien.grall@linaro.org>

Regards,
Oleksandr Tyshchenko March 13, 2014, 1:35 p.m. | #6
Hello, all.

I tested this patch (with debug patch) when I was looking for solution
to fix "simultaneous cross-interrupts" (see 1.1):
http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
This patch didn't fix my issue.
When the solution was founded I forgot about this patch)

Now I am using next solution:
http://marc.info/?l=xen-devel&m=139153089412362&w=2

But anyway, your patch works and looks good and I don't mind it.
Thank you.

Oleksandr Tyshchenko | Embedded Developer
GlobalLogic
www.globallogic.com

On Thu, Mar 13, 2014 at 2:56 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 03/13/2014 12:45 PM, Ian Campbell wrote:
>> On Thu, 2014-03-13 at 12:33 +0000, Julien Grall wrote:
>>> Hi Ian,
>>>
>>> On 01/28/2014 04:51 PM, Ian Campbell wrote:
>>>>      /* Set PPI and SGI priorities */
>>>> -    for (i = 0; i < 32; i += 4)
>>>> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
>>>> +    for (i = 0; i < 16; i += 4)
>>>> +        GICD[GICD_IPRIORITYR + i / 4] =
>>>> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
>>>> +    for (i = 16; i < 32; i += 4)
>>>> +        GICD[GICD_IPRIORITYR + i / 4] =
>>>> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>>>
>>> I'm wondering if it's necessary to set the priority for PPIs. It will be
>>> overridden later when the interrupt will be setup. Until that time, the
>>> interrupt is not enabled.
>>
>> Strictly speaking probably not, but there isn't much harm in setting a
>> sane default I suppose.
>
> Right, I didn't see that we also do that for SPIs.
>
> I have a minor change to request, can you divide the comment "Set PPI
> and SGI priorities" in 2 to reflect was does each loop?
>
> Except that:
>
> Acked-by: Julien Grall <julien.grall@linaro.org>
>
> Regards,
>
> --
> Julien Grall
Stefano Stabellini March 16, 2014, 4:38 p.m. | #7
On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
> Hello, all.
> 
> I tested this patch (with debug patch) when I was looking for solution
> to fix "simultaneous cross-interrupts" (see 1.1):
> http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
> This patch didn't fix my issue.
> When the solution was founded I forgot about this patch)
> 
> Now I am using next solution:
> http://marc.info/?l=xen-devel&m=139153089412362&w=2

Speaking of which, it should probably be applied to xen-unstable now
that the gate is open?

> But anyway, your patch works and looks good and I don't mind it.
> Thank you.
> 
> Oleksandr Tyshchenko | Embedded Developer
> GlobalLogic
> www.globallogic.com
> 
> On Thu, Mar 13, 2014 at 2:56 PM, Julien Grall <julien.grall@linaro.org> wrote:
> > On 03/13/2014 12:45 PM, Ian Campbell wrote:
> >> On Thu, 2014-03-13 at 12:33 +0000, Julien Grall wrote:
> >>> Hi Ian,
> >>>
> >>> On 01/28/2014 04:51 PM, Ian Campbell wrote:
> >>>>      /* Set PPI and SGI priorities */
> >>>> -    for (i = 0; i < 32; i += 4)
> >>>> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> >>>> +    for (i = 0; i < 16; i += 4)
> >>>> +        GICD[GICD_IPRIORITYR + i / 4] =
> >>>> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> >>>> +    for (i = 16; i < 32; i += 4)
> >>>> +        GICD[GICD_IPRIORITYR + i / 4] =
> >>>> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> >>>
> >>> I'm wondering if it's necessary to set the priority for PPIs. It will be
> >>> overridden later when the interrupt will be setup. Until that time, the
> >>> interrupt is not enabled.
> >>
> >> Strictly speaking probably not, but there isn't much harm in setting a
> >> sane default I suppose.
> >
> > Right, I didn't see that we also do that for SPIs.
> >
> > I have a minor change to request, can you divide the comment "Set PPI
> > and SGI priorities" in 2 to reflect was does each loop?
> >
> > Except that:
> >
> > Acked-by: Julien Grall <julien.grall@linaro.org>
> >
> > Regards,
> >
> > --
> > Julien Grall
> 
> 
> 
> -- 
> 
> Name | Title
> GlobalLogic
> P +x.xxx.xxx.xxxx  M +x.xxx.xxx.xxxx  S skype
> www.globallogic.com
> 
> http://www.globallogic.com/email_disclaimer.txt
>
Ian Campbell March 17, 2014, 10:02 a.m. | #8
On Sun, 2014-03-16 at 16:38 +0000, Stefano Stabellini wrote:
> On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
> > Hello, all.
> > 
> > I tested this patch (with debug patch) when I was looking for solution
> > to fix "simultaneous cross-interrupts" (see 1.1):
> > http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
> > This patch didn't fix my issue.
> > When the solution was founded I forgot about this patch)
> > 
> > Now I am using next solution:
> > http://marc.info/?l=xen-devel&m=139153089412362&w=2
> 
> Speaking of which, it should probably be applied to xen-unstable now
> that the gate is open?

Julien acked it but had a couple of minor comments which I wanted to
address today. I hope to repost later on.

Ian.
Ian Campbell March 17, 2014, 11:29 a.m. | #9
On Mon, 2014-03-17 at 10:02 +0000, Ian Campbell wrote:
> On Sun, 2014-03-16 at 16:38 +0000, Stefano Stabellini wrote:
> > On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
> > > Hello, all.
> > > 
> > > I tested this patch (with debug patch) when I was looking for solution
> > > to fix "simultaneous cross-interrupts" (see 1.1):
> > > http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
> > > This patch didn't fix my issue.
> > > When the solution was founded I forgot about this patch)
> > > 
> > > Now I am using next solution:
> > > http://marc.info/?l=xen-devel&m=139153089412362&w=2
> > 
> > Speaking of which, it should probably be applied to xen-unstable now
> > that the gate is open?
> 
> Julien acked it but had a couple of minor comments which I wanted to
> address today. I hope to repost later on.

Oh you meant the route irqs to dom0 patch in the link not the one in
this thread. I'll pick that one up today as well I hope.

Ian.
Julien Grall March 17, 2014, 11:34 a.m. | #10
Hi Ian,

On 03/17/2014 11:29 AM, Ian Campbell wrote:
> On Mon, 2014-03-17 at 10:02 +0000, Ian Campbell wrote:
>> On Sun, 2014-03-16 at 16:38 +0000, Stefano Stabellini wrote:
>>> On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
>>>> Hello, all.
>>>>
>>>> I tested this patch (with debug patch) when I was looking for solution
>>>> to fix "simultaneous cross-interrupts" (see 1.1):
>>>> http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
>>>> This patch didn't fix my issue.
>>>> When the solution was founded I forgot about this patch)
>>>>
>>>> Now I am using next solution:
>>>> http://marc.info/?l=xen-devel&m=139153089412362&w=2
>>>
>>> Speaking of which, it should probably be applied to xen-unstable now
>>> that the gate is open?
>>
>> Julien acked it but had a couple of minor comments which I wanted to
>> address today. I hope to repost later on.
> 
> Oh you meant the route irqs to dom0 patch in the link not the one in
> this thread. I'll pick that one up today as well I hope.

With the IPI patch, I don't think we need the route irqs to CPU0 patch
(at least for Xen 4.5). It would be better to backport it for Xen 4.4.

Regards,
Ian Campbell March 17, 2014, 11:36 a.m. | #11
On Mon, 2014-03-17 at 11:34 +0000, Julien Grall wrote:
> Hi Ian,
> 
> On 03/17/2014 11:29 AM, Ian Campbell wrote:
> > On Mon, 2014-03-17 at 10:02 +0000, Ian Campbell wrote:
> >> On Sun, 2014-03-16 at 16:38 +0000, Stefano Stabellini wrote:
> >>> On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
> >>>> Hello, all.
> >>>>
> >>>> I tested this patch (with debug patch) when I was looking for solution
> >>>> to fix "simultaneous cross-interrupts" (see 1.1):
> >>>> http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
> >>>> This patch didn't fix my issue.
> >>>> When the solution was founded I forgot about this patch)
> >>>>
> >>>> Now I am using next solution:
> >>>> http://marc.info/?l=xen-devel&m=139153089412362&w=2
> >>>
> >>> Speaking of which, it should probably be applied to xen-unstable now
> >>> that the gate is open?
> >>
> >> Julien acked it but had a couple of minor comments which I wanted to
> >> address today. I hope to repost later on.
> > 
> > Oh you meant the route irqs to dom0 patch in the link not the one in
> > this thread. I'll pick that one up today as well I hope.
> 
> With the IPI patch, I don't think we need the route irqs to CPU0 patch
> (at least for Xen 4.5). It would be better to backport it for Xen 4.4.

Oleksndr said above that the IPI patch didn't help but the CPU0 one did.
But rereading I wonder if I am misinterpreting what was said.

Ian.
Ian Campbell March 17, 2014, 12:42 p.m. | #12
On Mon, 2014-03-17 at 11:36 +0000, Ian Campbell wrote:
> On Mon, 2014-03-17 at 11:34 +0000, Julien Grall wrote:
> > Hi Ian,
> > 
> > On 03/17/2014 11:29 AM, Ian Campbell wrote:
> > > On Mon, 2014-03-17 at 10:02 +0000, Ian Campbell wrote:
> > >> On Sun, 2014-03-16 at 16:38 +0000, Stefano Stabellini wrote:
> > >>> On Thu, 13 Mar 2014, Oleksandr Tyshchenko wrote:
> > >>>> Hello, all.
> > >>>>
> > >>>> I tested this patch (with debug patch) when I was looking for solution
> > >>>> to fix "simultaneous cross-interrupts" (see 1.1):
> > >>>> http://lists.xenproject.org/archives/html/xen-devel/2014-01/msg02685.html
> > >>>> This patch didn't fix my issue.
> > >>>> When the solution was founded I forgot about this patch)
> > >>>>
> > >>>> Now I am using next solution:
> > >>>> http://marc.info/?l=xen-devel&m=139153089412362&w=2
> > >>>
> > >>> Speaking of which, it should probably be applied to xen-unstable now
> > >>> that the gate is open?
> > >>
> > >> Julien acked it but had a couple of minor comments which I wanted to
> > >> address today. I hope to repost later on.
> > > 
> > > Oh you meant the route irqs to dom0 patch in the link not the one in
> > > this thread. I'll pick that one up today as well I hope.
> > 
> > With the IPI patch, I don't think we need the route irqs to CPU0 patch
> > (at least for Xen 4.5). It would be better to backport it for Xen 4.4.
> 
> Oleksndr said above that the IPI patch didn't help but the CPU0 one did.
> But rereading I wonder if I am misinterpreting what was said.

Because of this uncertainty I'm going to wait for confirmation of what
is going on before I apply "xen/arm: route irqs to cpu0".

Ian.

Patch

diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index dcf9cd4..ee37019 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -319,7 +319,8 @@  static void __init gic_dist_init(void)
 
     /* Default priority for global interrupts */
     for ( i = 32; i < gic.lines; i += 4 )
-        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
 
     /* Disable all global interrupts */
     for ( i = 32; i < gic.lines; i += 32 )
@@ -341,8 +342,12 @@  static void __cpuinit gic_cpu_init(void)
     GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
     GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
     /* Set PPI and SGI priorities */
-    for (i = 0; i < 32; i += 4)
-        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
+    for (i = 0; i < 16; i += 4)
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
+    for (i = 16; i < 32; i += 4)
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
 
     /* Local settings: interface controller */
     GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
@@ -538,7 +543,8 @@  void gic_disable_cpu(void)
 void gic_route_ppis(void)
 {
     /* GIC maintenance */
-    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()), 0xa0);
+    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
+                     GIC_PRI_IRQ);
     /* Route timer interrupt */
     route_timer_interrupt();
 }
@@ -553,7 +559,8 @@  void gic_route_spis(void)
         if ( (irq = serial_dt_irq(seridx)) == NULL )
             continue;
 
-        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()), 0xa0);
+        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()),
+                         GIC_PRI_IRQ);
     }
 }
 
@@ -777,7 +784,7 @@  int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
     level = dt_irq_is_level_triggered(irq);
 
     gic_set_irq_properties(irq->irq, level, cpumask_of(smp_processor_id()),
-                           0xa0);
+                           GIC_PRI_IRQ);
 
     retval = __setup_irq(desc, irq->irq, action);
     if (retval) {
diff --git a/xen/arch/arm/time.c b/xen/arch/arm/time.c
index 81e3e28..68b939d 100644
--- a/xen/arch/arm/time.c
+++ b/xen/arch/arm/time.c
@@ -222,11 +222,11 @@  static void vtimer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
 void __cpuinit route_timer_interrupt(void)
 {
     gic_route_dt_irq(&timer_irq[TIMER_PHYS_NONSECURE_PPI],
-                     cpumask_of(smp_processor_id()), 0xa0);
+                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
     gic_route_dt_irq(&timer_irq[TIMER_HYP_PPI],
-                     cpumask_of(smp_processor_id()), 0xa0);
+                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
     gic_route_dt_irq(&timer_irq[TIMER_VIRT_PPI],
-                     cpumask_of(smp_processor_id()), 0xa0);
+                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
 }
 
 /* Set up the timer interrupt on this CPU */
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 9c6f9bb..25b2b24 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -129,6 +129,28 @@ 
 #define GICH_LR_CPUID_SHIFT     9
 #define GICH_VTR_NRLRGS         0x3f
 
+/*
+ * The minimum GICC_BPR is required to be in the range 0-3. We set
+ * GICC_BPR to 0 but we must expect that it might be 3. This means we
+ * can rely on premption between the following ranges:
+ * 0xf0..0xff
+ * 0xe0..0xdf
+ * 0xc0..0xcf
+ * 0xb0..0xbf
+ * 0xa0..0xaf
+ * 0x90..0x9f
+ * 0x80..0x8f
+ *
+ * Priorities within a range will not preempt each other.
+ *
+ * A GIC must support a mimimum of 16 priority levels.
+ */
+#define GIC_PRI_LOWEST     0xf0
+#define GIC_PRI_IRQ        0xa0
+#define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
+#define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
+
+
 #ifndef __ASSEMBLY__
 #include <xen/device_tree.h>