[ARM64,v4.4,V3,08/44] arm64: uaccess: Don't bother eliding access_ok checks in __{get, put}_user

Message ID 40fe1d91c9cd8d78ae952b821185681621f92b10.1567077734.git.viresh.kumar@linaro.org
State New
Headers show
Series
  • V4.4 backport of arm64 Spectre patches
Related show

Commit Message

Viresh Kumar Aug. 29, 2019, 11:33 a.m.
From: Will Deacon <will.deacon@arm.com>


commit 84624087dd7e3b482b7b11c170ebc1f329b3a218 upstream.

access_ok isn't an expensive operation once the addr_limit for the current
thread has been loaded into the cache. Given that the initial access_ok
check preceding a sequence of __{get,put}_user operations will take
the brunt of the miss, we can make the __* variants identical to the
full-fat versions, which brings with it the benefits of address masking.

The likely cost in these sequences will be from toggling PAN/UAO, which
we can address later by implementing the *_unsafe versions.

Reviewed-by: Robin Murphy <robin.murphy@arm.com>

Signed-off-by: Will Deacon <will.deacon@arm.com>

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

[ v4.4: Fixed conflicts around {__get_user|__put_user}_unaligned macros ]
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>

---
 arch/arm64/include/asm/uaccess.h | 62 ++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 26 deletions(-)

-- 
2.21.0.rc0.269.g1a574e7a288b

Comments

Mark Rutland Aug. 30, 2019, 9:41 a.m. | #1
On Thu, Aug 29, 2019 at 05:03:53PM +0530, Viresh Kumar wrote:
> From: Will Deacon <will.deacon@arm.com>

> 

> commit 84624087dd7e3b482b7b11c170ebc1f329b3a218 upstream.

> 

> access_ok isn't an expensive operation once the addr_limit for the current

> thread has been loaded into the cache. Given that the initial access_ok

> check preceding a sequence of __{get,put}_user operations will take

> the brunt of the miss, we can make the __* variants identical to the

> full-fat versions, which brings with it the benefits of address masking.

> 

> The likely cost in these sequences will be from toggling PAN/UAO, which

> we can address later by implementing the *_unsafe versions.

> 

> Reviewed-by: Robin Murphy <robin.murphy@arm.com>

> Signed-off-by: Will Deacon <will.deacon@arm.com>

> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

> [ v4.4: Fixed conflicts around {__get_user|__put_user}_unaligned macros ]

> Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>

> ---


Reviewed-by: Mark Rutland <mark.rutland@arm.com> [v4.4 backport]


Mark.

>  arch/arm64/include/asm/uaccess.h | 62 ++++++++++++++++++--------------

>  1 file changed, 36 insertions(+), 26 deletions(-)

> 

> diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h

> index fc11c50af558..a34324436ce1 100644

> --- a/arch/arm64/include/asm/uaccess.h

> +++ b/arch/arm64/include/asm/uaccess.h

> @@ -200,30 +200,35 @@ do {									\

>  			CONFIG_ARM64_PAN));				\

>  } while (0)

>  

> -#define __get_user(x, ptr)						\

> +#define __get_user_check(x, ptr, err)					\

>  ({									\

> -	int __gu_err = 0;						\

> -	__get_user_err((x), (ptr), __gu_err);				\

> -	__gu_err;							\

> +	__typeof__(*(ptr)) __user *__p = (ptr);				\

> +	might_fault();							\

> +	if (access_ok(VERIFY_READ, __p, sizeof(*__p))) {		\

> +		__p = uaccess_mask_ptr(__p);				\

> +		__get_user_err((x), __p, (err));			\

> +	} else {							\

> +		(x) = 0; (err) = -EFAULT;				\

> +	}								\

>  })

>  

>  #define __get_user_error(x, ptr, err)					\

>  ({									\

> -	__get_user_err((x), (ptr), (err));				\

> +	__get_user_check((x), (ptr), (err));				\

>  	(void)0;							\

>  })

>  

> -#define __get_user_unaligned __get_user

> -

> -#define get_user(x, ptr)						\

> +#define __get_user(x, ptr)						\

>  ({									\

> -	__typeof__(*(ptr)) __user *__p = (ptr);				\

> -	might_fault();							\

> -	access_ok(VERIFY_READ, __p, sizeof(*__p)) ?			\

> -		__p = uaccess_mask_ptr(__p), __get_user((x), __p) :	\

> -		((x) = 0, -EFAULT);					\

> +	int __gu_err = 0;						\

> +	__get_user_check((x), (ptr), __gu_err);				\

> +	__gu_err;							\

>  })

>  

> +#define __get_user_unaligned __get_user

> +

> +#define get_user	__get_user

> +

>  #define __put_user_asm(instr, reg, x, addr, err)			\

>  	asm volatile(							\

>  	"1:	" instr "	" reg "1, [%2]\n"			\

> @@ -266,30 +271,35 @@ do {									\

>  			CONFIG_ARM64_PAN));				\

>  } while (0)

>  

> -#define __put_user(x, ptr)						\

> +#define __put_user_check(x, ptr, err)					\

>  ({									\

> -	int __pu_err = 0;						\

> -	__put_user_err((x), (ptr), __pu_err);				\

> -	__pu_err;							\

> +	__typeof__(*(ptr)) __user *__p = (ptr);				\

> +	might_fault();							\

> +	if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) {		\

> +		__p = uaccess_mask_ptr(__p);				\

> +		__put_user_err((x), __p, (err));			\

> +	} else	{							\

> +		(err) = -EFAULT;					\

> +	}								\

>  })

>  

>  #define __put_user_error(x, ptr, err)					\

>  ({									\

> -	__put_user_err((x), (ptr), (err));				\

> +	__put_user_check((x), (ptr), (err));				\

>  	(void)0;							\

>  })

>  

> -#define __put_user_unaligned __put_user

> -

> -#define put_user(x, ptr)						\

> +#define __put_user(x, ptr)						\

>  ({									\

> -	__typeof__(*(ptr)) __user *__p = (ptr);				\

> -	might_fault();							\

> -	access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ?			\

> -		__p = uaccess_mask_ptr(__p), __put_user((x), __p) :	\

> -		-EFAULT;						\

> +	int __pu_err = 0;						\

> +	__put_user_check((x), (ptr), __pu_err);				\

> +	__pu_err;							\

>  })

>  

> +#define __put_user_unaligned __put_user

> +

> +#define put_user	__put_user

> +

>  extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);

>  extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);

>  extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);

> -- 

> 2.21.0.rc0.269.g1a574e7a288b

>

Patch

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index fc11c50af558..a34324436ce1 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -200,30 +200,35 @@  do {									\
 			CONFIG_ARM64_PAN));				\
 } while (0)
 
-#define __get_user(x, ptr)						\
+#define __get_user_check(x, ptr, err)					\
 ({									\
-	int __gu_err = 0;						\
-	__get_user_err((x), (ptr), __gu_err);				\
-	__gu_err;							\
+	__typeof__(*(ptr)) __user *__p = (ptr);				\
+	might_fault();							\
+	if (access_ok(VERIFY_READ, __p, sizeof(*__p))) {		\
+		__p = uaccess_mask_ptr(__p);				\
+		__get_user_err((x), __p, (err));			\
+	} else {							\
+		(x) = 0; (err) = -EFAULT;				\
+	}								\
 })
 
 #define __get_user_error(x, ptr, err)					\
 ({									\
-	__get_user_err((x), (ptr), (err));				\
+	__get_user_check((x), (ptr), (err));				\
 	(void)0;							\
 })
 
-#define __get_user_unaligned __get_user
-
-#define get_user(x, ptr)						\
+#define __get_user(x, ptr)						\
 ({									\
-	__typeof__(*(ptr)) __user *__p = (ptr);				\
-	might_fault();							\
-	access_ok(VERIFY_READ, __p, sizeof(*__p)) ?			\
-		__p = uaccess_mask_ptr(__p), __get_user((x), __p) :	\
-		((x) = 0, -EFAULT);					\
+	int __gu_err = 0;						\
+	__get_user_check((x), (ptr), __gu_err);				\
+	__gu_err;							\
 })
 
+#define __get_user_unaligned __get_user
+
+#define get_user	__get_user
+
 #define __put_user_asm(instr, reg, x, addr, err)			\
 	asm volatile(							\
 	"1:	" instr "	" reg "1, [%2]\n"			\
@@ -266,30 +271,35 @@  do {									\
 			CONFIG_ARM64_PAN));				\
 } while (0)
 
-#define __put_user(x, ptr)						\
+#define __put_user_check(x, ptr, err)					\
 ({									\
-	int __pu_err = 0;						\
-	__put_user_err((x), (ptr), __pu_err);				\
-	__pu_err;							\
+	__typeof__(*(ptr)) __user *__p = (ptr);				\
+	might_fault();							\
+	if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) {		\
+		__p = uaccess_mask_ptr(__p);				\
+		__put_user_err((x), __p, (err));			\
+	} else	{							\
+		(err) = -EFAULT;					\
+	}								\
 })
 
 #define __put_user_error(x, ptr, err)					\
 ({									\
-	__put_user_err((x), (ptr), (err));				\
+	__put_user_check((x), (ptr), (err));				\
 	(void)0;							\
 })
 
-#define __put_user_unaligned __put_user
-
-#define put_user(x, ptr)						\
+#define __put_user(x, ptr)						\
 ({									\
-	__typeof__(*(ptr)) __user *__p = (ptr);				\
-	might_fault();							\
-	access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ?			\
-		__p = uaccess_mask_ptr(__p), __put_user((x), __p) :	\
-		-EFAULT;						\
+	int __pu_err = 0;						\
+	__put_user_check((x), (ptr), __pu_err);				\
+	__pu_err;							\
 })
 
+#define __put_user_unaligned __put_user
+
+#define put_user	__put_user
+
 extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);
 extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);
 extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);