@@ -49,11 +49,7 @@ FUNCTION(grub_arm_clean_dcache_range_armv7)
@ Clean data cache for range to point-of-unification
1: cmp r0, r1
bge 2f
-#ifdef ARMV6
mcr p15, 0, r0, c7, c10, 1 @ Clean data cache line by MVA
-#else
- mcr p15, 0, r0, c7, c11, 1 @ DCCMVAU
-#endif
add r0, r0, r2 @ Next line
b 1b
2: DSB
@@ -79,45 +75,39 @@ FUNCTION(grub_arm_invalidate_icache_range_armv7)
bx lr
#ifdef ARMV6
-FUNCTION(grub_arm_disable_caches_mmu_armv6)
+FUNCTION(grub_arm_disable_dcache_mmu_and_boot_armv6)
#else
-FUNCTION(grub_arm_disable_caches_mmu_armv7)
+FUNCTION(grub_arm_disable_dcache_mmu_and_boot_armv7)
+#endif
+ @ preserve arguments - beyond point of no return, so no stacking needed
+ mov r4, r0
+ mov r5, r1
+ mov r6, r2
+
+ @ ensure that the instructions below can be fetched with the Dcache off
+ adr r0, 0f
+ adr r1, 1f
+ mov r2, #4 @ architecturally minimal Dlinesize
+#ifdef ARMV6
+ bl grub_arm_clean_dcache_range_armv6
+#else
+ bl grub_arm_clean_dcache_range_armv7
#endif
- push {r4, lr}
-
- @ disable D-cache
- mrc p15, 0, r0, c1, c0, 0
- bic r0, r0, #(1 << 2)
- mcr p15, 0, r0, c1, c0, 0
- DSB
- ISB
-
- @ clean/invalidate D-cache
- bl clean_invalidate_dcache
-
- @ disable I-cache
- mrc p15, 0, r0, c1, c0, 0
- bic r0, r0, #(1 << 12)
- mcr p15, 0, r0, c1, c0, 0
- DSB
- ISB
-
- @ invalidate I-cache (also invalidates branch predictors)
- mcr p15, 0, r0, c7, c5, 0
- DSB
- ISB
-
- @ clear SCTLR M bit
+ @ disable D-cache and MMU - I-cache can remain enabled
mrc p15, 0, r0, c1, c0, 0
- bic r0, r0, #(1 << 0)
+ bic r0, r0, #(1 << 2) | (1 << 0)
mcr p15, 0, r0, c1, c0, 0
-
- mcr p15, 0, r0, c8, c7, 0 @ invalidate TLB
- mcr p15, 0, r0, c7, c5, 6 @ invalidate branch predictor
- DSB
- ISB
-
- pop {r4, lr}
- bx lr
-
+0: ISB
+
+ /* Boot the kernel.
+ * Arguments to kernel:
+ * r0 - 0
+ * r1 - machine type
+ * r2 - address of DTB
+ */
+ mov r0, #0
+ mov r1, r5
+ mov r2, r6
+ bx r4
+1:
@@ -33,8 +33,12 @@ void grub_arm_invalidate_icache_range_armv6 (grub_addr_t start, grub_addr_t end,
grub_addr_t dlinesz);
void grub_arm_invalidate_icache_range_armv7 (grub_addr_t start, grub_addr_t end,
grub_addr_t dlinesz);
-void grub_arm_disable_caches_mmu_armv6 (void);
-void grub_arm_disable_caches_mmu_armv7 (void);
+void grub_arm_disable_dcache_mmu_and_boot_armv6 (grub_addr_t linux_addr,
+ grub_uint32_t machine_type,
+ void *fdt_addr);
+void grub_arm_disable_dcache_mmu_and_boot_armv7 (grub_addr_t linux_addr,
+ grub_uint32_t machine_type,
+ void *fdt_addr);
grub_uint32_t grub_arm_main_id (void);
grub_uint32_t grub_arm_cache_type (void);
@@ -253,7 +257,9 @@ grub_arch_sync_caches (void *address, grub_size_t len)
}
void
-grub_arm_disable_caches_mmu (void)
+grub_arm_disable_dcache_mmu_and_boot (grub_addr_t linux_addr,
+ grub_uint32_t machine_type,
+ void *fdt_addr)
{
if (type == ARCH_UNKNOWN)
probe_caches ();
@@ -262,10 +268,12 @@ grub_arm_disable_caches_mmu (void)
case ARCH_ARMV5_WRITE_THROUGH:
case ARCH_ARMV6_UNIFIED:
case ARCH_ARMV6:
- grub_arm_disable_caches_mmu_armv6 ();
+ grub_arm_disable_dcache_mmu_and_boot_armv6 (linux_addr, machine_type,
+ fdt_addr);
break;
case ARCH_ARMV7:
- grub_arm_disable_caches_mmu_armv7 ();
+ grub_arm_disable_dcache_mmu_and_boot_armv7 (linux_addr, machine_type,
+ fdt_addr);
break;
/* Pacify GCC. */
case ARCH_UNKNOWN:
@@ -33,94 +33,4 @@
# define ISB isb
#define ARMV7 1
- @ r0 - CLIDR
- @ r1 - LoC
- @ r2 - current level
- @ r3 - num sets
- @ r4 - num ways
- @ r5 - current set
- @ r6 - current way
- @ r7 - line size
- @ r8 - scratch
- @ r9 - scratch
- @ r10 - scratch
- @ r11 - scratch
-clean_invalidate_dcache:
- push {r4-r12, lr}
- mrc p15, 1, r0, c0, c0, 1 @ Read CLIDR
- lsr r1, r0, #24 @ Extract LoC
- and r1, r1, #0x7
-
- mov r2, #0 @ First level, L1
-2: and r8, r0, #7 @ cache type at current level
- cmp r8, #2
- blt 5f @ instruction only, or none, skip level
-
- @ set current cache level/type (for CCSIDR read)
- lsl r8, r2, #1
- mcr p15, 2, r8, c0, c0, 0 @ Write CSSELR (level, type: data/uni)
-
- @ read current cache information
- mrc p15, 1, r8, c0, c0, 0 @ Read CCSIDR
- lsr r3, r8, #13 @ Number of sets -1
-
- @ Keep only 14 bits of r3
- lsl r3, r3, #18
- lsr r3, r3, #18
-
- lsr r4, r8, #3 @ Number of ways -1
-
- @ Keep only 9 bits of r4
- lsl r4, r4, #23
- lsr r4, r4, #23
-
- and r7, r8, #7 @ log2(line size in words) - 2
- add r7, r7, #2 @ adjust
- mov r8, #1
- lsl r7, r8, r7 @ -> line size in words
- lsl r7, r7, #2 @ -> bytes
-
- @ set loop
- mov r5, #0 @ current set = 0
-3: lsl r8, r2, #1 @ insert level
- clz r9, r7 @ calculate set field offset
- mov r10, #31
- sub r9, r10, r9
- lsl r10, r5, r9
- orr r8, r8, r10 @ insert set field
-
- @ way loop
- @ calculate way field offset
- mov r6, #0 @ current way = 0
- add r10, r4, #1
- clz r9, r10 @ r9 = way field offset
- add r9, r9, #1
-4: lsl r10, r6, r9
- orr r11, r8, r10 @ insert way field
-
- @ clean and invalidate line by set/way
- mcr p15, 0, r11, c7, c14, 2 @ DCCISW
-
- @ next way
- add r6, r6, #1
- cmp r6, r4
- ble 4b
-
- @ next set
- add r5, r5, #1
- cmp r5, r3
- ble 3b
-
- @ next level
-5: lsr r0, r0, #3 @ align next level CLIDR 'type' field
- add r2, r2, #1 @ increment cache level counter
- cmp r2, r1
- blt 2b @ outer loop
-
- @ return
-6: DSB
- ISB
- pop {r4-r12, lr}
- bx lr
-
-#include "cache.S"
\ No newline at end of file
+#include "cache.S"
@@ -44,8 +44,6 @@ static char *linux_args;
static grub_uint32_t machine_type;
static void *fdt_addr;
-typedef void (*kernel_entry_t) (int, unsigned long, void *);
-
#define LINUX_ZIMAGE_OFFSET 0x24
#define LINUX_ZIMAGE_MAGIC 0x016f2818
@@ -142,9 +140,12 @@ linux_prepare_atag (void)
to += 2;
/* Copy updated FDT to its launch location */
- grub_memcpy (atag_orig, tmp_atag, sizeof (grub_uint32_t) * (to - tmp_atag));
+ tmp_size = sizeof (grub_uint32_t) * (to - tmp_atag);
+ grub_memcpy (atag_orig, tmp_atag, tmp_size);
grub_free (tmp_atag);
+ grub_arch_sync_caches (atag_orig, tmp_size);
+
grub_dprintf ("loader", "ATAG updated for Linux boot\n");
return GRUB_ERR_NONE;
@@ -212,6 +213,8 @@ linux_prepare_fdt (void)
grub_memcpy (fdt_addr, tmp_fdt, tmp_size);
grub_free (tmp_fdt);
+ grub_arch_sync_caches (fdt_addr, tmp_size);
+
grub_dprintf ("loader", "FDT updated for Linux boot\n");
return GRUB_ERR_NONE;
@@ -224,7 +227,6 @@ failure:
static grub_err_t
linux_boot (void)
{
- kernel_entry_t linuxmain;
int fdt_valid, atag_valid;
fdt_valid = (fdt_addr && grub_fdt_check_header_nosize (fdt_addr) == 0);
@@ -266,14 +268,6 @@ linux_boot (void)
grub_dprintf ("loader", "Jumping to Linux...\n");
- /* Boot the kernel.
- * Arguments to kernel:
- * r0 - 0
- * r1 - machine type
- * r2 - address of DTB
- */
- linuxmain = (kernel_entry_t) linux_addr;
-
#ifdef GRUB_MACHINE_EFI
{
grub_err_t err;
@@ -283,9 +277,7 @@ linux_boot (void)
}
#endif
- grub_arm_disable_caches_mmu ();
-
- linuxmain (0, machine_type, fdt_addr);
+ grub_arm_disable_dcache_mmu_and_boot (linux_addr, machine_type, fdt_addr);
return grub_error (GRUB_ERR_BAD_OS, "Linux call returned");
}
@@ -431,6 +423,8 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
if (grub_initrd_load (&initrd_ctx, argv, (void *) initrd_start))
goto fail;
+ grub_arch_sync_caches ((void *) initrd_start, size);
+
initrd_end = initrd_start + size;
return GRUB_ERR_NONE;
@@ -9,7 +9,10 @@ enum
GRUB_ARM_MACHINE_TYPE_FDT = 0xFFFFFFFF
};
-void EXPORT_FUNC(grub_arm_disable_caches_mmu) (void);
+void
+EXPORT_FUNC(grub_arm_disable_dcache_mmu_and_boot) (grub_addr_t linux_addr,
+ grub_uint32_t machine_type,
+ void *fdt_addr);
void grub_arm_enable_caches_mmu (void);
void grub_arm_enable_mmu (grub_uint32_t *mmu_tables);
void grub_arm_clear_mmu_v6 (void);
This removes the set/way cache maintenance for the ARM boot path, which is an inappropriate thing to do for GRUB [0]. To avoid the need for cache maintenance on the stack, refactor the boot path so that we no longer touch the stack after the MMU and caches have been disabled. Also, clean the FDT, ATAG array and/or the initrd to the PoC explicitly, so that we can remove the set/way maintenance altogether. (The kernel image itself is already cleaned to the PoC) Note that this requires grub_arm_clean_dcache_range_armv7() to be modified so that it cleans to the PoC rather than to the PoU, which makes it identical to the v6 version. [0] Set/way ops manage the state of the caches of a single core, which is typically done when it is powered up or down. Ensuring that certain memory stores have made it all the way to main memory is a completely different thing, and cannot be achieved on an ARM SMP system with set/way ops. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> --- Sending internally, to linaro-uefi@ and selected cc'ees. Gerd has confirmed that this patch solves his issue, but I have no clue whether the coding style or other GRUB-isms are done correctly here. grub-core/kern/arm/cache.S | 72 +++++++-------- grub-core/kern/arm/cache.c | 18 ++-- grub-core/kern/arm/cache_armv7.S | 92 +------------------- grub-core/loader/arm/linux.c | 24 ++--- include/grub/arm/system.h | 5 +- 5 files changed, 58 insertions(+), 153 deletions(-)