[Linaro-uefi] arm: remove set/way cache maintenance

Message ID 1487587909-3185-1-git-send-email-ard.biesheuvel@linaro.org
State New
Headers show

Commit Message

Ard Biesheuvel Feb. 20, 2017, 10:51 a.m.
This removes the set/way cache maintenance for the ARM boot path, which
is an inappropriate thing to do for GRUB [0]. To avoid the need for cache
maintenance on the stack, refactor the boot path so that we no longer
touch the stack after the MMU and caches have been disabled. Also, clean
the FDT, ATAG array and/or the initrd to the PoC explicitly, so that we
can remove the set/way maintenance altogether. (The kernel image itself
is already cleaned to the PoC)

Note that this requires grub_arm_clean_dcache_range_armv7() to be modified
so that it cleans to the PoC rather than to the PoU, which makes it
identical to the v6 version.

[0] Set/way ops manage the state of the caches of a single core, which is
    typically done when it is powered up or down. Ensuring that certain
    memory stores have made it all the way to main memory is a completely
    different thing, and cannot be achieved on an ARM SMP system with
    set/way ops.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---

Sending internally, to linaro-uefi@ and selected cc'ees. Gerd has
confirmed that this patch solves his issue, but I have no clue whether
the coding style or other GRUB-isms are done correctly here.
 
 grub-core/kern/arm/cache.S       | 72 +++++++--------
 grub-core/kern/arm/cache.c       | 18 ++--
 grub-core/kern/arm/cache_armv7.S | 92 +-------------------
 grub-core/loader/arm/linux.c     | 24 ++---
 include/grub/arm/system.h        |  5 +-
 5 files changed, 58 insertions(+), 153 deletions(-)

Patch hide | download patch | download mbox

diff --git a/grub-core/kern/arm/cache.S b/grub-core/kern/arm/cache.S
index 354a069fe67a..8960a8944c6f 100644
--- a/grub-core/kern/arm/cache.S
+++ b/grub-core/kern/arm/cache.S
@@ -49,11 +49,7 @@  FUNCTION(grub_arm_clean_dcache_range_armv7)
 	@ Clean data cache for range to point-of-unification
 1:	cmp	r0, r1
 	bge	2f
-#ifdef ARMV6
 	mcr	p15, 0, r0, c7, c10, 1	@ Clean data cache line by MVA
-#else
-	mcr	p15, 0, r0, c7, c11, 1	@ DCCMVAU
-#endif
 	add	r0, r0, r2		@ Next line
 	b	1b
 2:	DSB
@@ -79,45 +75,39 @@  FUNCTION(grub_arm_invalidate_icache_range_armv7)
 	bx	lr
 
 #ifdef ARMV6
-FUNCTION(grub_arm_disable_caches_mmu_armv6)
+FUNCTION(grub_arm_disable_dcache_mmu_and_boot_armv6)
 #else
-FUNCTION(grub_arm_disable_caches_mmu_armv7)
+FUNCTION(grub_arm_disable_dcache_mmu_and_boot_armv7)
+#endif
+	@ preserve arguments - beyond point of no return, so no stacking needed
+	mov	r4, r0
+	mov	r5, r1
+	mov	r6, r2
+
+	@ ensure that the instructions below can be fetched with the Dcache off
+	adr	r0, 0f
+	adr	r1, 1f
+	mov	r2, #4			@ architecturally minimal Dlinesize
+#ifdef ARMV6
+	bl	grub_arm_clean_dcache_range_armv6
+#else
+	bl	grub_arm_clean_dcache_range_armv7
 #endif
 
-	push	{r4, lr}
-
-	@ disable D-cache
-	mrc	p15, 0, r0, c1, c0, 0
-	bic	r0, r0, #(1 << 2)
-	mcr	p15, 0, r0, c1, c0, 0
-	DSB
-	ISB
-
-	@ clean/invalidate D-cache
-	bl	clean_invalidate_dcache
-
-	@ disable I-cache
-	mrc	p15, 0, r0, c1, c0, 0
-	bic	r0, r0, #(1 << 12)
-	mcr	p15, 0, r0, c1, c0, 0
-	DSB
-	ISB
-
-	@ invalidate I-cache (also invalidates branch predictors)
-	mcr	p15, 0, r0, c7, c5, 0
-	DSB
-	ISB
-
-	@ clear SCTLR M bit
+	@ disable D-cache and MMU - I-cache can remain enabled
 	mrc	p15, 0, r0, c1, c0, 0
-	bic	r0, r0, #(1 << 0)
+	bic	r0, r0, #(1 << 2) | (1 << 0)
 	mcr	p15, 0, r0, c1, c0, 0
-
-	mcr	p15, 0, r0, c8, c7, 0	@ invalidate TLB
-	mcr	p15, 0, r0, c7, c5, 6	@ invalidate branch predictor
-	DSB
-	ISB
-
-	pop	{r4, lr}
-	bx	lr
-
+0:	ISB
+
+	/* Boot the kernel.
+	 *   Arguments to kernel:
+	 *     r0 - 0
+	 *     r1 - machine type
+	 *     r2 - address of DTB
+	 */
+	mov	r0, #0
+	mov	r1, r5
+	mov	r2, r6
+	bx	r4
+1:
diff --git a/grub-core/kern/arm/cache.c b/grub-core/kern/arm/cache.c
index 34154ccdb0e0..eec17b00f3d1 100644
--- a/grub-core/kern/arm/cache.c
+++ b/grub-core/kern/arm/cache.c
@@ -33,8 +33,12 @@  void grub_arm_invalidate_icache_range_armv6 (grub_addr_t start, grub_addr_t end,
 					     grub_addr_t dlinesz);
 void grub_arm_invalidate_icache_range_armv7 (grub_addr_t start, grub_addr_t end,
 					     grub_addr_t dlinesz);
-void grub_arm_disable_caches_mmu_armv6 (void);
-void grub_arm_disable_caches_mmu_armv7 (void);
+void grub_arm_disable_dcache_mmu_and_boot_armv6 (grub_addr_t linux_addr,
+						 grub_uint32_t machine_type,
+						 void *fdt_addr);
+void grub_arm_disable_dcache_mmu_and_boot_armv7 (grub_addr_t linux_addr,
+						 grub_uint32_t machine_type,
+						 void *fdt_addr);
 grub_uint32_t grub_arm_main_id (void);
 grub_uint32_t grub_arm_cache_type (void);
 
@@ -253,7 +257,9 @@  grub_arch_sync_caches (void *address, grub_size_t len)
 }
 
 void
-grub_arm_disable_caches_mmu (void)
+grub_arm_disable_dcache_mmu_and_boot (grub_addr_t linux_addr,
+				      grub_uint32_t machine_type,
+				      void *fdt_addr)
 {
   if (type == ARCH_UNKNOWN)
     probe_caches ();
@@ -262,10 +268,12 @@  grub_arm_disable_caches_mmu (void)
     case ARCH_ARMV5_WRITE_THROUGH:
     case ARCH_ARMV6_UNIFIED:
     case ARCH_ARMV6:
-      grub_arm_disable_caches_mmu_armv6 ();
+      grub_arm_disable_dcache_mmu_and_boot_armv6 (linux_addr, machine_type,
+						  fdt_addr);
       break;
     case ARCH_ARMV7:
-      grub_arm_disable_caches_mmu_armv7 ();
+      grub_arm_disable_dcache_mmu_and_boot_armv7 (linux_addr, machine_type,
+						  fdt_addr);
       break;
       /* Pacify GCC.  */
     case ARCH_UNKNOWN:
diff --git a/grub-core/kern/arm/cache_armv7.S b/grub-core/kern/arm/cache_armv7.S
index 1ef2754af8a7..1d6597956ac8 100644
--- a/grub-core/kern/arm/cache_armv7.S
+++ b/grub-core/kern/arm/cache_armv7.S
@@ -33,94 +33,4 @@ 
 # define ISB	isb
 #define ARMV7 1
 
-	@ r0  - CLIDR
-	@ r1  - LoC
-	@ r2  - current level
-	@ r3  - num sets
-	@ r4  - num ways
-	@ r5  - current set
-	@ r6  - current way
-	@ r7  - line size
-	@ r8  - scratch
-	@ r9  - scratch
-	@ r10 - scratch
-	@ r11 - scratch
-clean_invalidate_dcache:
-	push	{r4-r12, lr}
-	mrc 	p15, 1, r0, c0, c0, 1	@ Read CLIDR
-	lsr	r1, r0, #24		@ Extract LoC
-	and	r1, r1, #0x7
-
-	mov	r2, #0			@ First level, L1
-2:	and	r8, r0, #7		@ cache type at current level
-	cmp	r8, #2
-	blt	5f			@ instruction only, or none, skip level
-
-	@ set current cache level/type (for CCSIDR read)
-	lsl	r8, r2, #1
-	mcr	p15, 2, r8, c0, c0, 0	@ Write CSSELR (level, type: data/uni)
-
-	@ read current cache information
-	mrc	p15, 1, r8, c0, c0, 0	@ Read CCSIDR
-	lsr	r3, r8, #13		@ Number of sets -1
-
-	@ Keep only 14 bits of r3
-	lsl     r3, r3, #18
-	lsr     r3, r3, #18
-
-	lsr	r4, r8, #3		@ Number of ways -1
-
-	@ Keep only 9  bits of r4
-	lsl     r4, r4, #23
-	lsr     r4, r4, #23
-
-	and	r7, r8, #7		@ log2(line size in words) - 2
-	add	r7, r7, #2		@  adjust
-	mov	r8, #1
-	lsl	r7, r8, r7		@  -> line size in words
-	lsl	r7, r7, #2		@  -> bytes
-
-	@ set loop
-	mov	r5, #0			@ current set = 0
-3:	lsl	r8, r2, #1		@ insert level
-	clz	r9, r7			@ calculate set field offset
-	mov	r10, #31
-	sub	r9, r10, r9
-	lsl	r10, r5, r9
-	orr	r8, r8, r10		@ insert set field
-
-	@ way loop
-	@ calculate way field offset
-	mov	r6, #0			@ current way = 0
-	add	r10, r4, #1
-	clz	r9, r10			@ r9 = way field offset
-	add	r9, r9, #1
-4:	lsl	r10, r6, r9
-	orr	r11, r8, r10		@ insert way field
-
-	@ clean and invalidate line by set/way
-	mcr	p15, 0, r11, c7, c14, 2	@ DCCISW
-
-	@ next way
-	add	r6, r6, #1
-	cmp	r6, r4
-	ble	4b
-
-	@ next set
-	add	r5, r5, #1
-	cmp	r5, r3
-	ble	3b
-
-	@ next level
-5:	lsr	r0, r0, #3		@ align next level CLIDR 'type' field
-	add	r2, r2, #1		@ increment cache level counter
-	cmp	r2, r1
-	blt	2b			@ outer loop
-
-	@ return
-6:	DSB
-	ISB
-	pop	{r4-r12, lr}
-	bx	lr
-
-#include "cache.S"
\ No newline at end of file
+#include "cache.S"
diff --git a/grub-core/loader/arm/linux.c b/grub-core/loader/arm/linux.c
index 5b39f02bb2e5..5e3914b7d9d2 100644
--- a/grub-core/loader/arm/linux.c
+++ b/grub-core/loader/arm/linux.c
@@ -44,8 +44,6 @@  static char *linux_args;
 static grub_uint32_t machine_type;
 static void *fdt_addr;
 
-typedef void (*kernel_entry_t) (int, unsigned long, void *);
-
 #define LINUX_ZIMAGE_OFFSET	0x24
 #define LINUX_ZIMAGE_MAGIC	0x016f2818
 
@@ -142,9 +140,12 @@  linux_prepare_atag (void)
   to += 2;
 
   /* Copy updated FDT to its launch location */
-  grub_memcpy (atag_orig, tmp_atag, sizeof (grub_uint32_t) * (to - tmp_atag));
+  tmp_size = sizeof (grub_uint32_t) * (to - tmp_atag);
+  grub_memcpy (atag_orig, tmp_atag, tmp_size);
   grub_free (tmp_atag);
 
+  grub_arch_sync_caches (atag_orig, tmp_size);
+
   grub_dprintf ("loader", "ATAG updated for Linux boot\n");
 
   return GRUB_ERR_NONE;
@@ -212,6 +213,8 @@  linux_prepare_fdt (void)
   grub_memcpy (fdt_addr, tmp_fdt, tmp_size);
   grub_free (tmp_fdt);
 
+  grub_arch_sync_caches (fdt_addr, tmp_size);
+
   grub_dprintf ("loader", "FDT updated for Linux boot\n");
 
   return GRUB_ERR_NONE;
@@ -224,7 +227,6 @@  failure:
 static grub_err_t
 linux_boot (void)
 {
-  kernel_entry_t linuxmain;
   int fdt_valid, atag_valid;
 
   fdt_valid = (fdt_addr && grub_fdt_check_header_nosize (fdt_addr) == 0);
@@ -266,14 +268,6 @@  linux_boot (void)
 
   grub_dprintf ("loader", "Jumping to Linux...\n");
 
-  /* Boot the kernel.
-   *   Arguments to kernel:
-   *     r0 - 0
-   *     r1 - machine type
-   *     r2 - address of DTB
-   */
-  linuxmain = (kernel_entry_t) linux_addr;
-
 #ifdef GRUB_MACHINE_EFI
   {
     grub_err_t err;
@@ -283,9 +277,7 @@  linux_boot (void)
   }
 #endif
 
-  grub_arm_disable_caches_mmu ();
-
-  linuxmain (0, machine_type, fdt_addr);
+  grub_arm_disable_dcache_mmu_and_boot (linux_addr, machine_type, fdt_addr);
 
   return grub_error (GRUB_ERR_BAD_OS, "Linux call returned");
 }
@@ -431,6 +423,8 @@  grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
   if (grub_initrd_load (&initrd_ctx, argv, (void *) initrd_start))
     goto fail;
 
+  grub_arch_sync_caches ((void *) initrd_start, size);
+
   initrd_end = initrd_start + size;
 
   return GRUB_ERR_NONE;
diff --git a/include/grub/arm/system.h b/include/grub/arm/system.h
index f62c18c13a87..5afce696a719 100644
--- a/include/grub/arm/system.h
+++ b/include/grub/arm/system.h
@@ -9,7 +9,10 @@  enum
     GRUB_ARM_MACHINE_TYPE_FDT = 0xFFFFFFFF
   };
 
-void EXPORT_FUNC(grub_arm_disable_caches_mmu) (void);
+void
+EXPORT_FUNC(grub_arm_disable_dcache_mmu_and_boot) (grub_addr_t linux_addr,
+						   grub_uint32_t machine_type,
+						   void *fdt_addr);
 void grub_arm_enable_caches_mmu (void);
 void grub_arm_enable_mmu (grub_uint32_t *mmu_tables);
 void grub_arm_clear_mmu_v6 (void);