From patchwork Wed Jul 12 22:32:58 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107534 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1411641qge; Wed, 12 Jul 2017 15:34:18 -0700 (PDT) X-Received: by 10.98.96.66 with SMTP id u63mr32662836pfb.68.1499898858664; Wed, 12 Jul 2017 15:34:18 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898858; cv=none; d=google.com; s=arc-20160816; b=l/IQA/oPhvmkBS7iXLJT87vNroPnVuLio3Q8cZPkYTeKeCIezN7o59KOay6CiTEr+w Iw3ajLlIeR1N04VsjIFVWepG4ng9jhiQ2XLkLAH5g28ug1HlTkB8CIJXfR8hgLkOzg3K /qC8EBda/f1jANCUZf7O7a0d6dcGiCjWHzV1d2hQCwkIPOhjYLhCMnU5kKPHs1KVtzz4 C9pQNveTxfOPCxxDfBVe/wn+LsyfjP05V7ayQNvhg2cwQmfzLu+qHRp/bWB5F9MJUpgk nmgn8YGRTFhVBp2/cXtfqlQjoM0uxLEFhiBbLmp4BG+ZHpnEWcnAL+WGHNKnD6XmQFZe o3BQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=QKtbmTfCL/Kfb0cdlsZfhaD1PJCgZCPNkeUhn1eJXZc=; b=RAXsd8Sng8FqZahc2KxayJeEmWqwPJZIanKengd6xAdLfs/8JTnjiLZXHvFRkmTaBX KDHqaWaGnPz0Y3QmzAZxCXrnyoefULUxzbq0HsZTWRVvfQHDx7XzqtrUboiq0BouAZvk BcuAHaPOaFljkmNWk23a5ZVK08RsQ4fUt5rwjOPhQA2y2OkY4HTtH4tm4YKHCDia5D8+ aIxsJZAp3IrkPq9vMX78UfyQSgbHNPsxbT8kr+74jZD3p9suuh4zdS+LbU/kXzfPvtbW J9n+G/yJFxewsnHhP467KwZUBIYpoKH85qZD9Qb3BAw3SQcqw+tIw0vYMrecj5nut6w3 oMiw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id y2si2966209pli.72.2017.07.12.15.34.18; Wed, 12 Jul 2017 15:34:18 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752417AbdGLWeI (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:08 -0400 Received: from foss.arm.com ([217.140.101.70]:59070 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751069AbdGLWeG (ORCPT ); Wed, 12 Jul 2017 18:34:06 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3454B1596; Wed, 12 Jul 2017 15:34:06 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 588163F3E1; Wed, 12 Jul 2017 15:34:04 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 1/6] arm64: use tpidr_el1 for current, free sp_el0 Date: Wed, 12 Jul 2017 23:32:58 +0100 Message-Id: <1499898783-25732-2-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Today we use TPIDR_EL1 for our percpu offset, and SP_EL0 for current (and current::thread_info, which is at offset 0). Using SP_EL0 in this way prevents us from using EL1 thread mode, where SP_EL0 is not addressable (since it's used as the active SP). It also means we can't use SP_EL0 for other purposes (e.g. as a scratch-register). This patch frees up SP_EL0 for such usage, by storing the percpu offset in current::thread_info, and using TPIDR_EL1 to store current. As we no longer need to update SP_EL0 at EL0 exception boundaries, this allows us to delete some code. This new organisation means that we need to perform an additional load to acquire the prcpu offset. However, our assembly constraints allow current to be cached, and therefore allow the offset to be cached. Additionally, in most cases where we need the percpu offset, we also need to fiddle with the preempt count or other data stored in current::thread_info, so this data should already be hot in the caches. Signed-off-by: Mark Rutland --- arch/arm64/include/asm/assembler.h | 11 ++++++++--- arch/arm64/include/asm/current.h | 6 +++--- arch/arm64/include/asm/percpu.h | 15 ++++----------- arch/arm64/include/asm/thread_info.h | 1 + arch/arm64/kernel/asm-offsets.c | 1 + arch/arm64/kernel/entry.S | 11 ++--------- arch/arm64/kernel/head.S | 4 ++-- arch/arm64/kernel/process.c | 16 ++++------------ 8 files changed, 25 insertions(+), 40 deletions(-) -- 1.9.1 diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 1b67c37..f7da6b5 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -229,6 +229,11 @@ #endif .endm + .macro get_this_cpu_offset dst + mrs \dst, tpidr_el1 + ldr \dst, [\dst, #TSK_TI_PCP] + .endm + /* * @dst: Result of per_cpu(sym, smp_processor_id()) * @sym: The name of the per-cpu variable @@ -236,7 +241,7 @@ */ .macro adr_this_cpu, dst, sym, tmp adr_l \dst, \sym - mrs \tmp, tpidr_el1 + get_this_cpu_offset \tmp add \dst, \dst, \tmp .endm @@ -247,7 +252,7 @@ */ .macro ldr_this_cpu dst, sym, tmp adr_l \dst, \sym - mrs \tmp, tpidr_el1 + get_this_cpu_offset \tmp ldr \dst, [\dst, \tmp] .endm @@ -438,7 +443,7 @@ * Return the current thread_info. */ .macro get_thread_info, rd - mrs \rd, sp_el0 + mrs \rd, tpidr_el1 .endm /* diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h index f6580d4..54b271a 100644 --- a/arch/arm64/include/asm/current.h +++ b/arch/arm64/include/asm/current.h @@ -13,11 +13,11 @@ */ static __always_inline struct task_struct *get_current(void) { - unsigned long sp_el0; + unsigned long cur; - asm ("mrs %0, sp_el0" : "=r" (sp_el0)); + asm ("mrs %0, tpidr_el1" : "=r" (cur)); - return (struct task_struct *)sp_el0; + return (struct task_struct *)cur; } #define current get_current() diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 3bd498e..05cf0f8 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -18,23 +18,16 @@ #include +#include + static inline void set_my_cpu_offset(unsigned long off) { - asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory"); + current_thread_info()->pcp_offset = off; } static inline unsigned long __my_cpu_offset(void) { - unsigned long off; - - /* - * We want to allow caching the value, so avoid using volatile and - * instead use a fake stack read to hazard against barrier(). - */ - asm("mrs %0, tpidr_el1" : "=r" (off) : - "Q" (*(const unsigned long *)current_stack_pointer)); - - return off; + return current_thread_info()->pcp_offset; } #define __my_cpu_offset __my_cpu_offset() diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 46c3b93..141f13e9 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -50,6 +50,7 @@ struct thread_info { #ifdef CONFIG_ARM64_SW_TTBR0_PAN u64 ttbr0; /* saved TTBR0_EL1 */ #endif + unsigned long pcp_offset; int preempt_count; /* 0 => preemptable, <0 => bug */ }; diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index b3bb7ef..17001be 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -38,6 +38,7 @@ int main(void) BLANK(); DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); + DEFINE(TSK_TI_PCP, offsetof(struct task_struct, thread_info.pcp_offset)); DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit)); #ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index b738880..773b3fea 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -92,7 +92,7 @@ .if \el == 0 mrs x21, sp_el0 - ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear, + get_thread_info tsk // Ensure MDSCR_EL1.SS is clear, ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug disable_step_tsk x19, x20 // exceptions when scheduling. @@ -147,13 +147,6 @@ alternative_else_nop_endif .endif /* - * Set sp_el0 to current thread_info. - */ - .if \el == 0 - msr sp_el0, tsk - .endif - - /* * Registers that may be useful after this macro is invoked: * * x21 - aborted SP @@ -734,7 +727,7 @@ ENTRY(cpu_switch_to) ldp x29, x9, [x8], #16 ldr lr, [x8] mov sp, x9 - msr sp_el0, x1 + msr tpidr_el1, x1 ret ENDPROC(cpu_switch_to) diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 973df7d..a58ecda 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -324,7 +324,7 @@ __primary_switched: adrp x4, init_thread_union add sp, x4, #THREAD_SIZE adr_l x5, init_task - msr sp_el0, x5 // Save thread_info + msr tpidr_el1, x5 // Save thread_info adr_l x8, vectors // load VBAR_EL1 with virtual msr vbar_el1, x8 // vector table address @@ -615,7 +615,7 @@ __secondary_switched: ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack mov sp, x1 ldr x2, [x0, #CPU_BOOT_TASK] - msr sp_el0, x2 + msr tpidr_el1, x2 mov x29, #0 b secondary_start_kernel ENDPROC(__secondary_switched) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index ae2a835..4212da3 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -323,18 +323,10 @@ void uao_thread_switch(struct task_struct *next) } } -/* - * We store our current task in sp_el0, which is clobbered by userspace. Keep a - * shadow copy so that we can restore this upon entry from userspace. - * - * This is *only* for exception entry from EL0, and is not valid until we - * __switch_to() a user task. - */ -DEFINE_PER_CPU(struct task_struct *, __entry_task); - -static void entry_task_switch(struct task_struct *next) +/* Ensure the new task has this CPU's offset */ +void pcp_thread_switch(struct task_struct *next) { - __this_cpu_write(__entry_task, next); + next->thread_info.pcp_offset = current_thread_info()->pcp_offset; } /* @@ -349,8 +341,8 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, tls_thread_switch(next); hw_breakpoint_thread_switch(next); contextidr_thread_switch(next); - entry_task_switch(next); uao_thread_switch(next); + pcp_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case From patchwork Wed Jul 12 22:32:59 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107536 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1411659qge; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) X-Received: by 10.84.237.8 with SMTP id s8mr6413364plk.163.1499898859612; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898859; cv=none; d=google.com; s=arc-20160816; b=XuO4/976PtSkGw6cliH6uurrzAjJ0/RQq179QIZEmX3w/gBjtrVUD/ZpbZZupwbuyT 5yGSephZicti9DEZoH/7MTHEsul4TIsBe47UjkV1VZ9hiRENPsXjQMjLzw+T5DIVofER L7Q2ln83USl09XmqBH0Yj7ibhjvdf1jQyk09wvgMbxlORhOzvwIPRSmslmMNUeGA4Qoi 1/cnrXhNmLX/Ro6fpq/xbs2tB25yguRPJMIvH5YdmYxemOz0Rx7yLcGIhbNhsTO1gHxn Hdd1rC19uP34C9ObCf5z83XPn9pSaRAEkFxRu1Z0pZXElS2aY5f6MA4wBk9kAp2iu/7L oIMQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=sTBRXUXcZSN6KKGRhyYJbKw/CSVp6UFy1fdfuNYKN4A=; b=TdKms7Wq5s8PzmUJmlpLGqOD1Sn7894nXT1cQTogwBAHGqZBUSY7nzXcooGn40RP2W 3RVZ51Xh1DKw5Ej8bD/KMgPLp2WLfHQ689CmnRpXHGNLqHxqiWMNwcdW4yW6Gxk5SU0K y+rEfDHaazpUVZK7gUxJEK08RVLYLg5oeKGbuxfmJ0kT4CBMAeQAG7uOEtxzWZR5iW71 uz9gKqvC05Dbgs38cwCMny6gRFTTlpaONfB3rZb0pJ0wUXBCh0WldaFB/qY0MSJiRofl hIanVD8mKkvhE3wIyzUWX6dGm2e6kfPKCx8nRD/gwiRG3Kcgy6x0fbyl4MkhIwGGR0is vTVw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id y2si2966209pli.72.2017.07.12.15.34.19; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752643AbdGLWeP (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:15 -0400 Received: from foss.arm.com ([217.140.101.70]:59086 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752523AbdGLWeO (ORCPT ); Wed, 12 Jul 2017 18:34:14 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id ABCE515B2; Wed, 12 Jul 2017 15:34:08 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id D36783F3E1; Wed, 12 Jul 2017 15:34:06 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 2/6] arm64: avoid open-coding THREAD_SIZE{,_ORDER} Date: Wed, 12 Jul 2017 23:32:59 +0100 Message-Id: <1499898783-25732-3-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Currently we define THREAD_SIZE_ORDER dependent on which arm64-specific page size kconfig symbol was selected. This is unfortunate, as it hides the relationship between THREAD_SIZE_ORDER and THREAD_SIZE, and makes it painful more painful than necessary to modify the thread size as we will need to do for some debug configurations. This patch follows arch/metag's approach of consistently defining THREAD_SIZE in terms of THREAD_SIZE_ORDER. This avoids having ifdefs for particular page size configurations, and allows us to change a single definition to change the thread size. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon --- arch/arm64/include/asm/thread_info.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) -- 1.9.1 diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 141f13e9..6d0c59a 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -23,13 +23,17 @@ #include -#ifdef CONFIG_ARM64_4K_PAGES -#define THREAD_SIZE_ORDER 2 -#elif defined(CONFIG_ARM64_16K_PAGES) +#include + +#define THREAD_SHIFT 14 + +#if THREAD_SHIFT >= PAGE_SHIFT +#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) +#else #define THREAD_SIZE_ORDER 0 #endif -#define THREAD_SIZE 16384 +#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define THREAD_START_SP (THREAD_SIZE - 16) #ifndef __ASSEMBLY__ From patchwork Wed Jul 12 22:33:00 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107535 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1411650qge; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) X-Received: by 10.98.163.203 with SMTP id q72mr56533770pfl.97.1499898859150; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898859; cv=none; d=google.com; s=arc-20160816; b=YHnR6rXe8VEeYAKZsXOHp0VUhQ/xDJhJOF/dzdek5H2L/ZUxDm2OcLViAmWNuEPd+f QFr6DFn+pKDVetXd+4s5XQCgBKieYKK9ep8b0RgoWK14bvSoesy+pZfiaDpZMiuz4GdR wGbjBTLq7TUU5ZDXp6Y+PFT0TskTJEHkq+62x53t5/gWm0makX6pq8IDJ7p6ar9XVWRb XrQz8ehh27lQSuyPv94Djk98hwwpWWpbxDtP9rNvq08E3Eo9O0OquKsUPa08l1kftQan dcrsjsO8orjREIpG+i2J7OkavoS7em7iu/tF/OcAqZVIqMGNORbTA01uMQbLK5x/c6fA 67Rw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=/dRvVKMXvmBfzhqq7UEBXZQNjBfDcqqaPT/5LdGTZho=; b=gM2mqWjzdZ3oRNwxPxKaVB3RPdSXql3UWvLtIT8rPeVFxw6cWjm6PTQdpTPUFo4e0W 1pJJM2qdPHzrS9oDy7vtoCj88z/2CJzb84LATMPYJJoBO8x+mNRSiw7HxBeyxVLBKlSb 9wNNmGjySe7Ovq5xxIaL2xmjS92MY3Z0XGZVFTNBMmGhvXR3kBPly8w0jpyKP5eLWYvB JncIj+FpNRH6KI8dJiaq0mSylu5fos8uD76S5IuSgOG62BLw6bHHpMRFro3Xa52lHvHr FUDaDyPqn+T8KLpnKHAZ7D2ZSGHQFFaRlGcy+uiIdq+xRv79j9kw5h831WuMwFXQ0Obl bX6Q== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id y2si2966209pli.72.2017.07.12.15.34.18; Wed, 12 Jul 2017 15:34:19 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752577AbdGLWeO (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:14 -0400 Received: from foss.arm.com ([217.140.101.70]:59100 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752428AbdGLWeL (ORCPT ); Wed, 12 Jul 2017 18:34:11 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1E81915BE; Wed, 12 Jul 2017 15:34:11 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 46ED33F3E1; Wed, 12 Jul 2017 15:34:09 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 3/6] arm64: pad stacks to PAGE_SIZE for VMAP_STACK Date: Wed, 12 Jul 2017 23:33:00 +0100 Message-Id: <1499898783-25732-4-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Our THREAD_SIZE may be smaller than PAGE_SIZE. With VMAP_STACK, we can't allow stacks to share a page with anything else, so may as well pad up-to PAGE_SIZE, and have 64K stacks when we have 64K pages. Signed-off-by: Mark Rutland --- arch/arm64/include/asm/thread_info.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) -- 1.9.1 diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 6d0c59a..3684f86 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -25,7 +25,13 @@ #include -#define THREAD_SHIFT 14 +#define __THREAD_SHIFT 14 + +#if defined(CONFIG_VMAP_STACK) && (__THREAD_SHIFT < PAGE_SHIFT) +#define THREAD_SHIFT PAGE_SHIFT +#else +#define THREAD_SHIFT __THREAD_SHIFT +#endif #if THREAD_SHIFT >= PAGE_SHIFT #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) From patchwork Wed Jul 12 22:33:01 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107538 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1411873qge; Wed, 12 Jul 2017 15:34:34 -0700 (PDT) X-Received: by 10.84.142.1 with SMTP id 1mr6610882plw.130.1499898873942; Wed, 12 Jul 2017 15:34:33 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898873; cv=none; d=google.com; s=arc-20160816; b=TchFE3w/Krj46+sNZqp2Y8Z6RsVpmrkv/O0AOxOoKwOqCIV1oI1gwcPnCH5Z5BmmCF M34bQXQ8ngdW42wEoWccY3EiWsp1AzDJHQRV+haS3gwCM6jjmmb1RH0DqelHXUgDqab9 qX1c5bjwIRJS2G27YnBA9zL3W7zkeRpOfwb/afw5Ecyu4cG+WtoPNqlL7IHvEEAWWNtC 8ipKuNWOH+zlqKOMsZYNycUohVNJ0MluQdQphMAx7IRSOxo/c9iRmCKbIMN/YUHsVyzd H+w36C5QRkWAa0ppSLL35YcAxKC+/Aw2dD1m15Pd7FRAOjIfOyzEIY6FNbCEMYwBGkmZ S4tg== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=0LLX/2NZsBgA08R0fM2IUHR90cWSG+tGsX5dhM9Xabo=; b=mTSclbXjvcCvYkDdyTA2lyEjkqPG87Q5en6fURKms38ETjiRriyvmZWoYKzGB/6IUu g/hOhXaUl0rCG05t8iF4SyWmxhCyH53sbQw/Z42DTB6Ua9VowSa+zYsW8K9H1i1GMnDt 10PMZm87vtmmqpyErk0jdXWYJf7WDaTKyPKAQL7LhQ58rqonWUXcVmHSF0u/Bsnzns6f 5z9yE/tNr0Y3e2MW4HNjFQFwQjhhr92VXoitWl70fxkGynMRefnz/cmOTdt5UGk33gzn ETtnGUyvWmW2tW5pWvx5bOHqvmQM4qRGQ5Y2aYYsFOLsYLpBCFPqDSuitTQSyDP57rq4 MqHw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id 3si2959805plp.163.2017.07.12.15.34.33; Wed, 12 Jul 2017 15:34:33 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752776AbdGLWeU (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:20 -0400 Received: from foss.arm.com ([217.140.101.70]:59114 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752523AbdGLWeT (ORCPT ); Wed, 12 Jul 2017 18:34:19 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 853C01610; Wed, 12 Jul 2017 15:34:13 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id AC2553F3E1; Wed, 12 Jul 2017 15:34:11 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 4/6] arm64: pass stack base to secondary_start_kernel Date: Wed, 12 Jul 2017 23:33:01 +0100 Message-Id: <1499898783-25732-5-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org In subsequent patches, we'll want the base of the secondary stack in secondary_start_kernel. Pass the stack base down, as we do in the primary path, and add the offset in secondary_start_kernel. Unfortunately, we can't encode STACK_START_SP in an add immediate, so use a mov immedaite, which has greater range. This is far from a hot path, so the overhead shouldn't matter. Signed-off-by: Mark Rutland --- arch/arm64/kernel/head.S | 3 ++- arch/arm64/kernel/smp.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) -- 1.9.1 diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index a58ecda..db77cac 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -613,7 +613,8 @@ __secondary_switched: adr_l x0, secondary_data ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack - mov sp, x1 + mov x3, #THREAD_START_SP + add sp, x1, x3 ldr x2, [x0, #CPU_BOOT_TASK] msr tpidr_el1, x2 mov x29, #0 diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 6e0e16a..269c957 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) * page tables. */ secondary_data.task = idle; - secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; + secondary_data.stack = task_stack_page(idle); update_cpu_boot_status(CPU_MMU_OFF); __flush_dcache_area(&secondary_data, sizeof(secondary_data)); From patchwork Wed Jul 12 22:33:02 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107537 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1411864qge; Wed, 12 Jul 2017 15:34:33 -0700 (PDT) X-Received: by 10.99.45.193 with SMTP id t184mr6205151pgt.209.1499898873580; Wed, 12 Jul 2017 15:34:33 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898873; cv=none; d=google.com; s=arc-20160816; b=AzNG6lEDRvAixwmEE9oxbbNWyA7H7b1Hoy9iwQJXx284/wo/Us8k9u+yP6pCM54b7w hgDayULWs8/6PBP4MhmG2AVWcHZQrqLNbo4mfFKTaqWxrknrNfw9sndjBQHu1F/FCDU2 bbI5Tc+KnyyuBbub4rL2rHudAqwYe035rjLto6v2nyzlN9OWNc62qV6gdbbJ8UHJGLo4 ycUJtACTZo5Xrdlab3/oIo35AxHEljXkM4PptjzaPil3U54C7Mbok9zF2EXYh2nFoUIz iRf9lodrcY6B3YDj3PU3pwvHJnKalkKWMQkf3lkKW4XXSASBXJMdYEkHpNOtYDSHJAat O0cA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=7EyuFeGwty1MDglq47PCyDi6zyCIg7PrwwpWMKKRfvY=; b=wJdbkW+SRZwxRxLXDQS3vDHUqVqAUR/RvfsqEo6+IdP1fS9BaOr5iueVH6hhC4HeVX PudVJdU20LySWxYfPtu4KcZIl6AEVW3pzEUMO1Kublfh3ZDgMSe7N5NMcmfXXxwCmz83 K4F+ih2z1Kj1R2QP833efkscNishIgy7EWB2/Or4J0ZJ3sZ7/laNDbgIoWVlrsecwDiL XOpTIeoh11534XOJMCZjgwNsMexeg3bKvhy/uIrxTbTBHQH86er44W6HGgfsSgq7CN9m snxQkbo33rrXDT0badzSg7vlM/XM6LJDae/nLOt2xOltVE+8XAGaS3mLvtWL8jrxUJMN DHtQ== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id 3si2959805plp.163.2017.07.12.15.34.33; Wed, 12 Jul 2017 15:34:33 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752721AbdGLWeS (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:18 -0400 Received: from foss.arm.com ([217.140.101.70]:59118 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752523AbdGLWeQ (ORCPT ); Wed, 12 Jul 2017 18:34:16 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id DE36D164F; Wed, 12 Jul 2017 15:34:15 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 0E6C23F3E1; Wed, 12 Jul 2017 15:34:13 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 5/6] arm64: keep track of current stack Date: Wed, 12 Jul 2017 23:33:02 +0100 Message-Id: <1499898783-25732-6-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org To reliably check stack bounds, we'll need to know whether we're on a task stack, or an IRQ stack. Stash the base of the current stack in thread_info so that we have this information. Signed-off-by: Mark Rutland --- arch/arm64/include/asm/thread_info.h | 3 +++ arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 7 +++++++ arch/arm64/kernel/head.S | 6 ++++++ arch/arm64/kernel/process.c | 4 ++++ 5 files changed, 23 insertions(+) -- 1.9.1 diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 3684f86..ae4f44b 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -62,6 +62,9 @@ struct thread_info { #endif unsigned long pcp_offset; int preempt_count; /* 0 => preemptable, <0 => bug */ +#ifdef CONFIG_VMAP_STACK + unsigned long current_stack; +#endif }; #define INIT_THREAD_INFO(tsk) \ diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 17001be..10c8ffa 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -40,6 +40,9 @@ int main(void) DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); DEFINE(TSK_TI_PCP, offsetof(struct task_struct, thread_info.pcp_offset)); DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit)); +#ifdef CONFIG_VMAP_STACK + DEFINE(TSK_TI_CUR_STK, offsetof(struct task_struct, thread_info.current_stack)); +#endif #ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); #endif diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 773b3fea..7c8b164 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -258,6 +258,9 @@ alternative_else_nop_endif /* switch to the irq stack */ mov sp, x26 +#ifdef CONFIG_VMAP_STACK + str x25, [tsk, #TSK_TI_CUR_STK] +#endif /* * Add a dummy stack frame, this non-standard format is fixed up @@ -275,6 +278,10 @@ alternative_else_nop_endif */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_VMAP_STACK + and x19, x19, #~(THREAD_SIZE - 1) + str x19, [tsk, #TSK_TI_CUR_STK] +#endif .endm /* diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index db77cac..3363846 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -325,6 +325,9 @@ __primary_switched: add sp, x4, #THREAD_SIZE adr_l x5, init_task msr tpidr_el1, x5 // Save thread_info +#ifdef CONFIG_VMAP_STACK + str x4, [x5, #TSK_TI_CUR_STK] +#endif adr_l x8, vectors // load VBAR_EL1 with virtual msr vbar_el1, x8 // vector table address @@ -616,6 +619,9 @@ __secondary_switched: mov x3, #THREAD_START_SP add sp, x1, x3 ldr x2, [x0, #CPU_BOOT_TASK] +#ifdef CONFIG_VMAP_STACK + str x1, [x2, #TSK_TI_CUR_STK] +#endif msr tpidr_el1, x2 mov x29, #0 b secondary_start_kernel diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 4212da3..5dc5797 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -294,6 +294,10 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, ptrace_hw_copy_thread(p); +#ifdef CONFIG_VMAP_STACK + p->thread_info.current_stack = (unsigned long)p->stack; +#endif + return 0; } From patchwork Wed Jul 12 22:33:03 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Rutland X-Patchwork-Id: 107539 Delivered-To: patch@linaro.org Received: by 10.140.101.44 with SMTP id t41csp1412135qge; Wed, 12 Jul 2017 15:34:50 -0700 (PDT) X-Received: by 10.98.13.219 with SMTP id 88mr58860006pfn.191.1499898889892; Wed, 12 Jul 2017 15:34:49 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1499898889; cv=none; d=google.com; s=arc-20160816; b=dIiH9ZJvHyoh2VN59EFlPZOXAvkm1VQNw/eBRT92qcnrVxEeWSTtrTR4wF7FDZfeDI hAiMkwGEk9PZX2DhB+rugM+b/c0++kxZdH61kNsHiST3a5Ehgeh18yxYy35az0xJHpsv T7dvZo6haK5KBzO5NB6zGunBI2lYh9xoSfGZvReUyvLekkXxoPYKiIeCokMo6HrN9HbO mHElFPcGpnkvaWHU1z715q58s8b61AeaZ64rM5rsImB7sX+GIYvTsYZDN0q+9tiNwH0P TJQGtIlVjI7tvkHzNYg1yPYaynyaT0+bCAtdaHsQmfY+iISoDjetdS6d1iGAR+6OAiwi W/Hg== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:references:in-reply-to:message-id:date :subject:cc:to:from:arc-authentication-results; bh=lu70MI5Id7Kqvpq0jS1OI00LkZk+Tter51ms4Hfk8+c=; b=TJdLnjGenuR8ApT+Yj2Lds6y+9DcUI1wANEjICK7kA3gTjg9WXWemjz0clYOVgsdLD +r2+mU1aU76PoU+XssZ0AGWfDngoDCZY7Uu2aa2IvY7bdZtEgtOyGoYEdb6LwMOTOCAG oWgjdVBPj8c+a5fY6R9mwSa+P8njCbj68Le23NcDpP6ImW+YN+EAzp7EI9ns5To2YmcE w34zGoey9WhOskRLs6oBhAcQmUT+HqGCfq5Jx+uo6VoqblgU0mOaQR0LQMZEd0nZXOoo pRtirwyhKUZWjCy3qWjMeWf6z2z5z29Odvw+2fJ+4HAx1mVpqhhHBAV5YOMp00x58ZS5 NSIA== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id 31si2955233plk.116.2017.07.12.15.34.49; Wed, 12 Jul 2017 15:34:49 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752833AbdGLWeb (ORCPT + 25 others); Wed, 12 Jul 2017 18:34:31 -0400 Received: from foss.arm.com ([217.140.101.70]:59132 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751883AbdGLWe2 (ORCPT ); Wed, 12 Jul 2017 18:34:28 -0400 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 6F8041650; Wed, 12 Jul 2017 15:34:18 -0700 (PDT) Received: from leverpostej.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.72.51.249]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 8EF303F3E1; Wed, 12 Jul 2017 15:34:16 -0700 (PDT) From: Mark Rutland To: ard.biesheuvel@linaro.org, kernel-hardening@lists.openwall.com, linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: akashi.takahiro@linaro.org, catalin.marinas@arm.com, dave.martin@arm.com, james.morse@arm.com, labbott@fedoraproject.org, will.deacon@arm.com, keescook@chromium.org, Mark Rutland Subject: [RFC PATCH 6/6] arm64: add VMAP_STACK and detect out-of-bounds SP Date: Wed, 12 Jul 2017 23:33:03 +0100 Message-Id: <1499898783-25732-7-git-send-email-mark.rutland@arm.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> References: <1499898783-25732-1-git-send-email-mark.rutland@arm.com> Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Signed-off-by: Mark Rutland --- arch/arm64/Kconfig | 1 + arch/arm64/kernel/entry.S | 43 +++++++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/traps.c | 21 +++++++++++++++++++++ 3 files changed, 65 insertions(+) -- 1.9.1 diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b2024db..5cbd961 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1,5 +1,6 @@ config ARM64 def_bool y + select HAVE_ARCH_VMAP_STACK select ACPI_CCA_REQUIRED if ACPI select ACPI_GENERIC_GSI if ACPI select ACPI_GTDT if ACPI diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 7c8b164..e0fdb65 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -396,11 +396,54 @@ el1_error_invalid: inv_entry 1, BAD_ERROR ENDPROC(el1_error_invalid) +#ifdef CONFIG_VMAP_STACK +.macro detect_bad_stack + msr sp_el0, x0 + get_thread_info x0 + ldr x0, [x0, #TSK_TI_CUR_STK] + sub x0, sp, x0 + and x0, x0, #~(THREAD_SIZE - 1) + cbnz x0, __bad_stack + mrs x0, sp_el0 +.endm + +__bad_stack: + /* + * Stash the bad SP, and free up another GPR. We no longer care about + * EL0 state, since this thread cannot recover. + */ + mov x0, sp + msr tpidrro_el0, x0 + msr tpidr_el0, x1 + + /* Move to the emergency stack */ + adr_this_cpu x0, bad_stack, x1 + mov x1, #THREAD_START_SP + add sp, x0, x1 + + /* Restore GPRs and log them to pt_regs */ + mrs x0, sp_el0 + mrs x1, tpidr_el0 + kernel_entry 1 + + /* restore the bad SP to pt_regs */ + mrs x1, tpidrro_el0 + str x1, [sp, #S_SP] + + /* Time to die */ + mov x0, sp + b handle_bad_stack +#else +.macro detect_bad_stack +.endm +#endif + /* * EL1 mode handlers. */ .align 6 el1_sync: + detect_bad_stack kernel_entry 1 mrs x1, esr_el1 // read the syndrome register lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 0805b44..84b00e3 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -683,6 +683,27 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) force_sig_info(info.si_signo, &info, current); } +#ifdef CONFIG_VMAP_STACK +DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], bad_stack) __aligned(16); + +asmlinkage void handle_bad_stack(struct pt_regs *regs) +{ + unsigned long tsk_stk = (unsigned long)current->stack; + unsigned long irq_stk = (unsigned long)per_cpu(irq_stack, smp_processor_id()); + + console_verbose(); + pr_emerg("Stack out-of-bounds!\n" + "\tsp: 0x%016lx\n" + "\ttsk stack: [0x%016lx..0x%016lx]\n" + "\tirq stack: [0x%016lx..0x%016lx]\n", + kernel_stack_pointer(regs), + tsk_stk, tsk_stk + THREAD_SIZE, + irq_stk, irq_stk + THREAD_SIZE); + show_regs(regs); + panic("stack out-of-bounds"); +} +#endif + void __pte_error(const char *file, int line, unsigned long val) { pr_err("%s:%d: bad pte %016lx.\n", file, line, val);