diff mbox series

[v5,5/7] arm64: head: Clean the ID map and the HYP text to the PoC if needed

Message ID 20221108182204.2447664-6-ardb@kernel.org
State Superseded
Headers show
Series arm64: efi: leave MMU and caches on at boot | expand

Commit Message

Ard Biesheuvel Nov. 8, 2022, 6:22 p.m. UTC
If we enter with the MMU and caches enabled, the bootloader may not have
performed any cache maintenance to the PoC. So clean the ID mapped page
to the PoC, to ensure that instruction and data accesses with the MMU
off see the correct data. For similar reasons, clean all the HYP text to
the PoC as well when entering at EL2 with the MMU and caches enabled.

Note that this means primary_entry() itself needs to be moved into the
ID map as well, as we will return from init_kernel_el() with the MMU and
caches off.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/kernel/head.S  | 31 +++++++++++++++++---
 arch/arm64/kernel/sleep.S |  1 +
 2 files changed, 28 insertions(+), 4 deletions(-)

Comments

Ard Biesheuvel Nov. 8, 2022, 10:11 p.m. UTC | #1
On Tue, 8 Nov 2022 at 19:22, Ard Biesheuvel <ardb@kernel.org> wrote:
>
> If we enter with the MMU and caches enabled, the bootloader may not have
> performed any cache maintenance to the PoC. So clean the ID mapped page
> to the PoC, to ensure that instruction and data accesses with the MMU
> off see the correct data. For similar reasons, clean all the HYP text to
> the PoC as well when entering at EL2 with the MMU and caches enabled.
>
> Note that this means primary_entry() itself needs to be moved into the
> ID map as well, as we will return from init_kernel_el() with the MMU and
> caches off.
>
> Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
> ---
>  arch/arm64/kernel/head.S  | 31 +++++++++++++++++---
>  arch/arm64/kernel/sleep.S |  1 +
>  2 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index a7c84cde67c5c652..825f1d0549661030 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -70,7 +70,7 @@
>
>         __EFI_PE_HEADER
>
> -       __INIT
> +       .section ".idmap.text","awx"
>
>         /*
>          * The following callee saved general purpose registers are used on the
> @@ -90,6 +90,17 @@ SYM_CODE_START(primary_entry)
>         bl      record_mmu_state
>         bl      preserve_boot_args
>         bl      create_idmap
> +
> +       /*
> +        * If we entered with the MMU and caches on, clean the ID mapped part
> +        * of the primary boot code to the PoC so we can safely execute it with
> +        * the MMU off.
> +        */
> +       cbz     x19, 0f
> +       adrp    x0, __idmap_text_start
> +       adr_l   x1, __idmap_text_end
> +       bl      dcache_clean_poc
> +0:     mov     x19, x0

This is wrong, it should be

mov x0, x19


>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mov     x20, x0
>
> @@ -111,6 +122,7 @@ SYM_CODE_START(primary_entry)
>         b       __primary_switch
>  SYM_CODE_END(primary_entry)
>
> +       __INIT
>  SYM_CODE_START_LOCAL(record_mmu_state)
>         mrs     x19, CurrentEL
>         cmp     x19, #CurrentEL_EL2
> @@ -505,10 +517,12 @@ SYM_FUNC_END(__primary_switched)
>   * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if
>   * booted in EL1 or EL2 respectively, with the top 32 bits containing
>   * potential context flags. These flags are *not* stored in __boot_cpu_mode.
> + *
> + * x0: whether we are being called from the primary boot path with the MMU on
>   */
>  SYM_FUNC_START(init_kernel_el)
> -       mrs     x0, CurrentEL
> -       cmp     x0, #CurrentEL_EL2
> +       mrs     x1, CurrentEL
> +       cmp     x1, #CurrentEL_EL2
>         b.eq    init_el2
>
>  SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
> @@ -523,6 +537,14 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
>         eret
>
>  SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
> +       msr     elr_el2, lr
> +
> +       // clean all HYP code to the PoC if we booted at EL2 with the MMU on
> +       cbz     x0, 0f
> +       adrp    x0, __hyp_idmap_text_start
> +       adr_l   x1, __hyp_text_end
> +       bl      dcache_clean_poc
> +0:
>         mov_q   x0, HCR_HOST_NVHE_FLAGS
>         msr     hcr_el2, x0
>         isb
> @@ -556,7 +578,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
>         msr     sctlr_el1, x1
>         mov     x2, xzr
>  2:
> -       msr     elr_el2, lr
>         mov     w0, #BOOT_CPU_MODE_EL2
>         orr     x0, x0, x2
>         eret
> @@ -567,6 +588,7 @@ SYM_FUNC_END(init_kernel_el)
>          * cores are held until we're ready for them to initialise.
>          */
>  SYM_FUNC_START(secondary_holding_pen)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mrs     x2, mpidr_el1
>         mov_q   x1, MPIDR_HWID_BITMASK
> @@ -584,6 +606,7 @@ SYM_FUNC_END(secondary_holding_pen)
>          * be used where CPUs are brought online dynamically by the kernel.
>          */
>  SYM_FUNC_START(secondary_entry)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         b       secondary_startup
>  SYM_FUNC_END(secondary_entry)
> diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
> index 7b7c56e048346e97..2ae7cff1953aaf87 100644
> --- a/arch/arm64/kernel/sleep.S
> +++ b/arch/arm64/kernel/sleep.S
> @@ -99,6 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter)
>
>         .pushsection ".idmap.text", "awx"
>  SYM_CODE_START(cpu_resume)
> +       mov     x0, xzr
>         bl      init_kernel_el
>         mov     x19, x0                 // preserve boot mode
>  #if VA_BITS > 48
> --
> 2.35.1
>
diff mbox series

Patch

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index a7c84cde67c5c652..825f1d0549661030 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -70,7 +70,7 @@ 
 
 	__EFI_PE_HEADER
 
-	__INIT
+	.section ".idmap.text","awx"
 
 	/*
 	 * The following callee saved general purpose registers are used on the
@@ -90,6 +90,17 @@  SYM_CODE_START(primary_entry)
 	bl	record_mmu_state
 	bl	preserve_boot_args
 	bl	create_idmap
+
+	/*
+	 * If we entered with the MMU and caches on, clean the ID mapped part
+	 * of the primary boot code to the PoC so we can safely execute it with
+	 * the MMU off.
+	 */
+	cbz	x19, 0f
+	adrp	x0, __idmap_text_start
+	adr_l	x1, __idmap_text_end
+	bl	dcache_clean_poc
+0:	mov	x19, x0
 	bl	init_kernel_el			// w0=cpu_boot_mode
 	mov	x20, x0
 
@@ -111,6 +122,7 @@  SYM_CODE_START(primary_entry)
 	b	__primary_switch
 SYM_CODE_END(primary_entry)
 
+	__INIT
 SYM_CODE_START_LOCAL(record_mmu_state)
 	mrs	x19, CurrentEL
 	cmp	x19, #CurrentEL_EL2
@@ -505,10 +517,12 @@  SYM_FUNC_END(__primary_switched)
  * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if
  * booted in EL1 or EL2 respectively, with the top 32 bits containing
  * potential context flags. These flags are *not* stored in __boot_cpu_mode.
+ *
+ * x0: whether we are being called from the primary boot path with the MMU on
  */
 SYM_FUNC_START(init_kernel_el)
-	mrs	x0, CurrentEL
-	cmp	x0, #CurrentEL_EL2
+	mrs	x1, CurrentEL
+	cmp	x1, #CurrentEL_EL2
 	b.eq	init_el2
 
 SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
@@ -523,6 +537,14 @@  SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
 	eret
 
 SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
+	msr	elr_el2, lr
+
+	// clean all HYP code to the PoC if we booted at EL2 with the MMU on
+	cbz	x0, 0f
+	adrp	x0, __hyp_idmap_text_start
+	adr_l	x1, __hyp_text_end
+	bl	dcache_clean_poc
+0:
 	mov_q	x0, HCR_HOST_NVHE_FLAGS
 	msr	hcr_el2, x0
 	isb
@@ -556,7 +578,6 @@  SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
 	msr	sctlr_el1, x1
 	mov	x2, xzr
 2:
-	msr	elr_el2, lr
 	mov	w0, #BOOT_CPU_MODE_EL2
 	orr	x0, x0, x2
 	eret
@@ -567,6 +588,7 @@  SYM_FUNC_END(init_kernel_el)
 	 * cores are held until we're ready for them to initialise.
 	 */
 SYM_FUNC_START(secondary_holding_pen)
+	mov	x0, xzr
 	bl	init_kernel_el			// w0=cpu_boot_mode
 	mrs	x2, mpidr_el1
 	mov_q	x1, MPIDR_HWID_BITMASK
@@ -584,6 +606,7 @@  SYM_FUNC_END(secondary_holding_pen)
 	 * be used where CPUs are brought online dynamically by the kernel.
 	 */
 SYM_FUNC_START(secondary_entry)
+	mov	x0, xzr
 	bl	init_kernel_el			// w0=cpu_boot_mode
 	b	secondary_startup
 SYM_FUNC_END(secondary_entry)
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index 7b7c56e048346e97..2ae7cff1953aaf87 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -99,6 +99,7 @@  SYM_FUNC_END(__cpu_suspend_enter)
 
 	.pushsection ".idmap.text", "awx"
 SYM_CODE_START(cpu_resume)
+	mov	x0, xzr
 	bl	init_kernel_el
 	mov	x19, x0			// preserve boot mode
 #if VA_BITS > 48