@@ -103,7 +103,7 @@ static inline unsigned long efi_get_kimg_min_align(void)
#define EFI_ALLOC_ALIGN SZ_64K
#define EFI_ALLOC_LIMIT ((1UL << 48) - 1)
-extern unsigned long primary_entry_offset(void);
+extern unsigned long entry_offset(void);
/*
* On ARM systems, virtually remapped UEFI runtime services are set up in two
@@ -95,7 +95,7 @@ zboot-obj-$(CONFIG_KERNEL_ZSTD) := zboot-decompress-zstd.o lib-xxhash.o
CFLAGS_zboot-decompress-zstd.o += -I$(srctree)/lib/zstd
zboot-obj-$(CONFIG_RISCV) += lib-clz_ctz.o lib-ashldi3.o
-lib-$(CONFIG_EFI_ZBOOT) += zboot.o $(zboot-obj-y)
+lib-$(CONFIG_EFI_ZBOOT) += zboot.o zboot-decompress.o $(zboot-obj-y)
lib-$(CONFIG_UNACCEPTED_MEMORY) += unaccepted_memory.o bitmap.o find.o
@@ -8,9 +8,15 @@ quiet_cmd_copy_and_pad = PAD $@
cmd_copy_and_pad = cp $< $@; \
truncate -s $$(hexdump -s16 -n4 -e '"%u"' $<) $@
+ifneq ($(EFI_ZBOOT_PAYLOAD),)
# Pad the file to the size of the uncompressed image in memory, including BSS
$(obj)/vmlinux.bin: $(obj)/$(EFI_ZBOOT_PAYLOAD) FORCE
$(call if_changed,copy_and_pad)
+else
+$(obj)/vmlinux.bin: OBJCOPYFLAGS := -R .note -R .note.gnu.build-id -R .comment -S
+$(obj)/vmlinux.bin: vmlinux FORCE
+ $(call if_changed,objcopy)
+endif
# in GZIP, the appended le32 carrying the uncompressed size is part of the
# format, but in other cases, we just append it at the end for convenience,
@@ -45,7 +45,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
asmlinkage void primary_entry(void);
-unsigned long primary_entry_offset(void)
+unsigned long entry_offset(void)
{
/*
* When built as part of the kernel, the EFI stub cannot branch to the
@@ -100,7 +100,7 @@ void efi_cache_sync_image(unsigned long image_base,
/* only perform the cache maintenance if needed for I/D coherency */
if (!(ctr & BIT(CTR_EL0_IDC_SHIFT))) {
unsigned long base = image_base;
- unsigned long size = code_size;
+ unsigned long size = alloc_size;
do {
asm("dc " DCTYPE ", %0" :: "r"(base));
@@ -116,24 +116,12 @@ void efi_cache_sync_image(unsigned long image_base,
efi_remap_image(image_base, alloc_size, code_size);
}
-unsigned long __weak primary_entry_offset(void)
-{
- /*
- * By default, we can invoke the kernel via the branch instruction in
- * the image header, so offset #0. This will be overridden by the EFI
- * stub build that is linked into the core kernel, as in that case, the
- * image header may not have been loaded into memory, or may be mapped
- * with non-executable permissions.
- */
- return 0;
-}
-
void __noreturn efi_enter_kernel(unsigned long entrypoint,
unsigned long fdt_addr,
unsigned long fdt_size)
{
void (* __noreturn enter_kernel)(u64, u64, u64, u64);
- enter_kernel = (void *)entrypoint + primary_entry_offset();
+ enter_kernel = (void *)entrypoint + entry_offset();
enter_kernel(fdt_addr, 0, 0, 0);
}
@@ -1232,7 +1232,13 @@ void process_unaccepted_memory(u64 start, u64 end);
void accept_memory(phys_addr_t start, unsigned long size);
void arch_accept_memory(phys_addr_t start, phys_addr_t end);
-efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size);
+efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size,
+ unsigned long *entry);
efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen);
+bool efi_zboot_check_header(unsigned long *alloc_size,
+ unsigned long *entry);
+bool efi_zboot_decompress_segments(u8 *out, unsigned long outlen);
+bool efi_zboot_decompress_slice(u8 *out, unsigned long outlen);
+
#endif
@@ -12,11 +12,11 @@
#include "inflate.c"
extern unsigned char _gzdata_start[], _gzdata_end[];
-extern u32 __aligned(1) payload_size;
static struct z_stream_s stream;
-efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size)
+efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size,
+ unsigned long *entry)
{
efi_status_t status;
int rc;
@@ -38,14 +38,18 @@ efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size)
goto out;
}
- *alloc_size = payload_size;
+ if (!efi_zboot_check_header(alloc_size, entry)) {
+ status = EFI_LOAD_ERROR;
+ goto out;
+ }
+
return EFI_SUCCESS;
out:
efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace);
return status;
}
-efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen)
+bool efi_zboot_decompress_slice(u8 *out, unsigned long outlen)
{
int rc;
@@ -53,12 +57,19 @@ efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen)
stream.avail_out = outlen;
rc = zlib_inflate(&stream, 0);
- zlib_inflateEnd(&stream);
+ return rc == Z_OK || rc == Z_STREAM_END;
+}
+
+efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen)
+{
+ bool ret = efi_zboot_decompress_segments(out, outlen);
+
+ zlib_inflateEnd(&stream);
efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace);
- if (rc != Z_STREAM_END) {
- efi_err("GZIP decompression failed with status %d\n", rc);
+ if (!ret) {
+ efi_err("GZIP decompression failed\n");
return EFI_LOAD_ERROR;
}
@@ -9,14 +9,14 @@
#include "efistub.h"
extern unsigned char _gzdata_start[], _gzdata_end[];
-extern u32 __aligned(1) payload_size;
static ZSTD_inBuffer zstd_buf;
static ZSTD_DStream *dstream;
static size_t wksp_size;
static void *wksp;
-efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size)
+efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size,
+ unsigned long *entry)
{
zstd_frame_header header;
efi_status_t status;
@@ -51,29 +51,39 @@ efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size)
goto out;
}
- *alloc_size = payload_size;
+ if (!efi_zboot_check_header(alloc_size, entry)) {
+ status = EFI_LOAD_ERROR;
+ goto out;
+ }
+
return EFI_SUCCESS;
out:
efi_free(wksp_size, (unsigned long)wksp);
return status;
}
-efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen)
+bool efi_zboot_decompress_slice(u8 *out, unsigned long outlen)
{
ZSTD_outBuffer zstd_dec;
size_t ret;
- int retval;
zstd_dec.dst = out;
zstd_dec.pos = 0;
zstd_dec.size = outlen;
ret = zstd_decompress_stream(dstream, &zstd_dec, &zstd_buf);
+
+ return zstd_get_error_code(ret) == 0;
+}
+
+efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen)
+{
+ bool ret = efi_zboot_decompress_segments(out, outlen);
+
efi_free(wksp_size, (unsigned long)wksp);
- retval = zstd_get_error_code(ret);
- if (retval) {
- efi_err("ZSTD-decompression failed with status %d\n", retval);
+ if (!ret) {
+ efi_err("ZSTD-decompression failed\n");
return EFI_LOAD_ERROR;
}
new file mode 100644
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/efi.h>
+#include <linux/elf.h>
+
+#include <asm/efi.h>
+
+#include "efistub.h"
+
+extern u32 __aligned(1) payload_size;
+
+static struct {
+#ifdef CONFIG_64BIT
+ Elf64_Ehdr ehdr;
+ Elf64_Phdr phdr[5];
+#else
+ Elf32_Ehdr ehdr;
+ Elf32_Phdr phdr[5];
+#endif
+} elf_header;
+
+static bool is_elf;
+
+bool efi_zboot_check_header(unsigned long *alloc_size,
+ unsigned long *entry)
+{
+ unsigned long min = ULONG_MAX, max = 0;
+ bool ret;
+
+ ret = efi_zboot_decompress_slice((u8 *)&elf_header, sizeof(elf_header));
+ if (!ret) {
+ efi_err("failed to extract header\n");
+ return false;
+ }
+
+ /* Check the ELF magic */
+ if (elf_header.ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+ elf_header.ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+ elf_header.ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+ elf_header.ehdr.e_ident[EI_MAG3] != ELFMAG3) {
+ /*
+ * Raw images are padded to the memory size before compression,
+ * so the payload size equals the allocation size.
+ */
+ *alloc_size = payload_size;
+ *entry = 0;
+ return true;
+ }
+
+ /*
+ * Check whether the executable header and program headers are laid out
+ * as expected.
+ */
+ if (elf_header.ehdr.e_phoff != offsetof(typeof(elf_header), phdr) ||
+ elf_header.ehdr.e_phnum > ARRAY_SIZE(elf_header.phdr)) {
+ efi_err("Unexpected ELF header layout\n");
+ return false;
+ }
+
+ /*
+ * Iterate over the PT_LOAD headers to find the size of the executable
+ * image in memory.
+ */
+ for (int i = 0; i < elf_header.ehdr.e_phnum; i++) {
+ __auto_type ph = &elf_header.phdr[i];
+
+ if (ph->p_type != PT_LOAD)
+ continue;
+
+ min = min(min, ph->p_paddr);
+ max = max(max, ph->p_paddr + ph->p_memsz);
+ }
+
+ if (min >= max) {
+ efi_err("Failed to determine ELF memory size\n");
+ return false;
+ }
+
+ efi_info("ELF zboot payload detected\n");
+
+ *alloc_size = max - min;
+ *entry = elf_header.ehdr.e_entry - elf_header.phdr[0].p_paddr;
+ is_elf = true;
+
+ return true;
+}
+
+bool efi_zboot_decompress_segments(u8 *out, unsigned long outlen)
+{
+ efi_memory_attribute_protocol_t *memattr = NULL;
+ unsigned long pos = sizeof(elf_header);
+
+ if (!is_elf) {
+ /*
+ * If this is a raw image, first copy the data we already
+ * extracted from the compressed blob into the output.
+ */
+ memcpy(out, &elf_header, pos);
+
+ return efi_zboot_decompress_slice(out + pos, outlen - pos);
+ }
+
+ /* grab a reference to the memory attributes protocol, if available */
+ efi_bs_call(locate_protocol, &EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID, NULL,
+ (void **)&memattr);
+
+ /*
+ * Iterate over the program headers, and decompress the payload of each
+ * PT_LOAD entry. This involves skipping the padding by decompressing
+ * it into the output buffer before overwriting it with the actual
+ * data.
+ */
+ for (int i = 0; i < elf_header.ehdr.e_phnum; i++) {
+ __auto_type ph = &elf_header.phdr[i];
+ void *dst = out + ph->p_paddr - elf_header.phdr[0].p_paddr;
+ unsigned long pa = (unsigned long)dst;
+
+ if (ph->p_type != PT_LOAD)
+ continue;
+
+ if (ph->p_offset < pos) {
+ efi_err("ELF PT_LOAD headers out of order\n");
+ return false;
+ }
+
+ /* extract and discard the padding */
+ while (ph->p_offset > pos) {
+ unsigned long l = min(ph->p_offset - pos, ph->p_memsz);
+
+ efi_zboot_decompress_slice(dst, l);
+ pos += l;
+ }
+
+ /* decompress payload */
+ efi_zboot_decompress_slice(dst, ph->p_filesz);
+
+ /* clear area that was not covered by file data */
+ if (ph->p_memsz > ph->p_filesz)
+ memset(dst + ph->p_filesz, 0, ph->p_memsz - ph->p_filesz);
+
+ if (memattr && ph->p_flags == (PF_R | PF_X)) {
+ unsigned long l = ALIGN(ph->p_memsz, EFI_PAGE_SIZE);
+ efi_status_t status;
+
+ status = memattr->set_memory_attributes(memattr, pa, l, EFI_MEMORY_RO);
+ if (status != EFI_SUCCESS)
+ efi_warn("Failed to set EFI_MEMORY_RO on R-X region\n");
+
+ status = memattr->clear_memory_attributes(memattr, pa, l, EFI_MEMORY_XP);
+ if (status != EFI_SUCCESS)
+ efi_warn("Failed to clear EFI_MEMORY_XP from R-X region\n");
+ }
+
+ if (ph->p_flags & PF_X)
+ efi_cache_sync_image(pa, ph->p_filesz);
+
+ pos = ph->p_offset + ph->p_filesz;
+ }
+
+ return true;
+}
@@ -31,6 +31,13 @@ struct screen_info *alloc_screen_info(void)
return __alloc_screen_info();
}
+static unsigned long entrypoint;
+
+unsigned long entry_offset(void)
+{
+ return entrypoint;
+}
+
asmlinkage efi_status_t __efiapi
efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab)
{
@@ -54,7 +61,7 @@ efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab)
efi_info("Decompressing Linux Kernel...\n");
- status = efi_zboot_decompress_init(&alloc_size);
+ status = efi_zboot_decompress_init(&alloc_size, &entrypoint);
if (status != EFI_SUCCESS)
return status;