diff mbox series

[v3,3/3] s390/crypto: New s390 specific shash phmac

Message ID 20241107145521.424769-4-freude@linux.ibm.com
State Superseded
Headers show
Series New s390 specific protected key hmac | expand

Commit Message

Harald Freudenberger Nov. 7, 2024, 2:55 p.m. UTC 
From: Holger Dengler <dengler@linux.ibm.com>

Add support for protected key hmac ("phmac") for s390 arch.

With the latest machine generation there is now support for
protected key (that is a key wrapped by a master key stored
in firmware) hmac for sha2 (sha224, sha256, sha384 and sha512)
for the s390 specific CPACF instruction kmac.

This patch adds support via 4 new shashes registered as
phmac(sha224), phmac(sha256), phmac(sha384) and phmac(sha512).

Please note that as of now, there is no selftest enabled for
these shashes, but the implementation has been tested with
testcases via AF_ALG interface.

Signed-off-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
---
 arch/s390/configs/debug_defconfig |   1 +
 arch/s390/configs/defconfig       |   1 +
 arch/s390/crypto/Makefile         |   1 +
 arch/s390/crypto/phmac_s390.c     | 473 ++++++++++++++++++++++++++++++
 drivers/crypto/Kconfig            |  12 +
 5 files changed, 488 insertions(+)
 create mode 100644 arch/s390/crypto/phmac_s390.c

Comments

Ingo Franzki Nov. 11, 2024, 12:13 p.m. UTC | #1 
On 07.11.2024 15:55, Harald Freudenberger wrote:
> From: Holger Dengler <dengler@linux.ibm.com>
> 
> Add support for protected key hmac ("phmac") for s390 arch.
> 
> With the latest machine generation there is now support for
> protected key (that is a key wrapped by a master key stored
> in firmware) hmac for sha2 (sha224, sha256, sha384 and sha512)
> for the s390 specific CPACF instruction kmac.
> 
> This patch adds support via 4 new shashes registered as
> phmac(sha224), phmac(sha256), phmac(sha384) and phmac(sha512).
> 
> Please note that as of now, there is no selftest enabled for
> these shashes, but the implementation has been tested with
> testcases via AF_ALG interface.
> 
> Signed-off-by: Holger Dengler <dengler@linux.ibm.com>
> Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
> ---
>  arch/s390/configs/debug_defconfig |   1 +
>  arch/s390/configs/defconfig       |   1 +
>  arch/s390/crypto/Makefile         |   1 +
>  arch/s390/crypto/phmac_s390.c     | 473 ++++++++++++++++++++++++++++++
>  drivers/crypto/Kconfig            |  12 +
>  5 files changed, 488 insertions(+)
>  create mode 100644 arch/s390/crypto/phmac_s390.c
> 
> diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
> index 6b602d972e91..a8ba72e7162d 100644
> --- a/arch/s390/configs/debug_defconfig
> +++ b/arch/s390/configs/debug_defconfig
> @@ -804,6 +804,7 @@ CONFIG_PKEY_EP11=m
>  CONFIG_PKEY_PCKMO=m
>  CONFIG_PKEY_UV=m
>  CONFIG_CRYPTO_PAES_S390=m
> +CONFIG_CRYPTO_PHMAC_S390=m
>  CONFIG_CRYPTO_DEV_VIRTIO=m
>  CONFIG_SYSTEM_BLACKLIST_KEYRING=y
>  CONFIG_CORDIC=m
> diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
> index 7844b9f5851b..cd2e9c94b736 100644
> --- a/arch/s390/configs/defconfig
> +++ b/arch/s390/configs/defconfig
> @@ -790,6 +790,7 @@ CONFIG_PKEY_EP11=m
>  CONFIG_PKEY_PCKMO=m
>  CONFIG_PKEY_UV=m
>  CONFIG_CRYPTO_PAES_S390=m
> +CONFIG_CRYPTO_PHMAC_S390=m
>  CONFIG_CRYPTO_DEV_VIRTIO=m
>  CONFIG_SYSTEM_BLACKLIST_KEYRING=y
>  CONFIG_CORDIC=m
> diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile
> index a0cb96937c3d..47637140b95c 100644
> --- a/arch/s390/crypto/Makefile
> +++ b/arch/s390/crypto/Makefile
> @@ -16,6 +16,7 @@ obj-$(CONFIG_S390_PRNG) += prng.o
>  obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
>  obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o
>  obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
> +obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o
>  obj-y += arch_random.o
>  
>  crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o
> diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c
> new file mode 100644
> index 000000000000..77a5244b2eb4
> --- /dev/null
> +++ b/arch/s390/crypto/phmac_s390.c
> @@ -0,0 +1,473 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * Copyright IBM Corp. 2024
> + *
> + * s390 specific HMAC support for protected keys.
> + */
> +
> +#define KMSG_COMPONENT	"phmac_s390"
> +#define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
> +
> +#include <asm/cpacf.h>
> +#include <asm/pkey.h>
> +#include <crypto/sha2.h>
> +#include <crypto/internal/hash.h>
> +#include <linux/cpufeature.h>
> +#include <linux/delay.h>
> +#include <linux/module.h>
> +#include <linux/spinlock.h>
> +
> +/*
> + * KMAC param block layout for sha2 function codes:
> + * The layout of the param block for the KMAC instruction depends on the
> + * blocksize of the used hashing sha2-algorithm function codes. The param block
> + * contains the hash chaining value (cv), the input message bit-length (imbl)
> + * and the hmac-secret (key). To prevent code duplication, the sizes of all
> + * these are calculated based on the blocksize.
> + *
> + * param-block:
> + * +-------+
> + * | cv    |
> + * +-------+
> + * | imbl  |
> + * +-------+
> + * | key   |
> + * +-------+
> + *
> + * sizes:
> + * part | sh2-alg | calculation | size | type
> + * -----+---------+-------------+------+--------
> + * cv	| 224/256 | blocksize/2 |   32 |  u64[8]
> + *	| 384/512 |		|   64 | u128[8]
> + * imbl | 224/256 | blocksize/8 |    8 |     u64
> + *	| 384/512 |		|   16 |    u128
> + * key	| 224/256 | blocksize	|   96 |  u8[96]
> + *	| 384/512 |		|  160 | u8[160]
> + */
> +
> +#define MAX_DIGEST_SIZE		SHA512_DIGEST_SIZE
> +#define MAX_IMBL_SIZE		sizeof(u128)
> +#define MAX_BLOCK_SIZE		SHA512_BLOCK_SIZE
> +
> +#define SHA2_CV_SIZE(bs)	((bs) >> 1)
> +#define SHA2_IMBL_SIZE(bs)	((bs) >> 3)
> +
> +#define SHA2_IMBL_OFFSET(bs)	(SHA2_CV_SIZE(bs))
> +#define SHA2_KEY_OFFSET(bs)	(SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))
> +
> +#define PHMAC_SHA256_KEY_SIZE	(SHA256_BLOCK_SIZE + 32)
> +#define PHMAC_SHA512_KEY_SIZE	(SHA512_BLOCK_SIZE + 32)
> +#define PHMAC_MAX_KEY_SIZE	PHMAC_SHA512_KEY_SIZE
> +
> +struct phmac_protkey {
> +	u32 type;
> +	u32 len;
> +	u8 protkey[PHMAC_MAX_KEY_SIZE];
> +};
> +
> +struct s390_phmac_ctx {
> +	u8 *key;
> +	unsigned int keylen;
> +
> +	struct phmac_protkey pk;
> +	/* spinlock to atomic update pk */
> +	spinlock_t pk_lock;
> +};
> +

....


> +
> +static inline int s390_phmac_sha2_setkey(struct crypto_shash *tfm,
> +					 const u8 *key, unsigned int keylen)
> +{
> +	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(tfm);
> +
> +	if (tfm_ctx->keylen) {
> +		kfree_sensitive(tfm_ctx->key);
> +		tfm_ctx->key = NULL;
> +		tfm_ctx->keylen = 0;
> +	}
> +
> +	tfm_ctx->key = kmemdup(key, keylen, GFP_ATOMIC);
> +	if (!tfm_ctx->key)
> +		return -ENOMEM;
> +	tfm_ctx->keylen = keylen;
> +
> +	return 0;
> +}
> +
> +static int s390_phmac_sha2_init(struct shash_desc *desc)
> +{
> +	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
> +	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
> +	unsigned int bs = crypto_shash_blocksize(desc->tfm);
> +	int rc;
> +
> +	rc = phmac_convert_key(desc->tfm);

There is no need to perform the (possibly time consuming) key conversation at every init.
You only need to convert the key if a new key was set (via setkey function) before. 
Once converted, it can be reused by multiple init calls, if no other setkey is done in between.

One possibility to achieve this is to add a flag field into struct s390_phmac_ctx (i.e. new_key_set).
In the setkey function set new_key_set to 1 once te key has been updated in the context. 
In the init function, if new_key_set is != 0, then convert the key and set new_key_set to 0. 
Any subsequent init calls (with no setkey in-between) will see new_key_set == 0 and thus will not convert the key again, but reuse the already converted key.

If the key hasn't changed it is simply unnecessary to perform the (possibly time consuming) key conversation again and again.

This improves performance for use cases where a tfm is allocated, a key is set once, and then multiple sequences of init/update/final are performed to calculate the HMACs of different pieces of data but with the same key. This is for example what dm-integrity does, it does init/update/final for every sector, but the key is set only once at the very beginning. 

> +	if (rc)
> +		goto out;
> +
> +	spin_lock_bh(&tfm_ctx->pk_lock);
> +	memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
> +	       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
> +	spin_unlock_bh(&tfm_ctx->pk_lock);
> +
> +	ctx->buflen = 0;
> +	ctx->gr0.reg = 0;
> +
> +	switch (crypto_shash_digestsize(desc->tfm)) {
> +	case SHA224_DIGEST_SIZE:
> +		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_224;
> +		break;
> +	case SHA256_DIGEST_SIZE:
> +		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_256;
> +		break;
> +	case SHA384_DIGEST_SIZE:
> +		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_384;
> +		break;
> +	case SHA512_DIGEST_SIZE:
> +		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_512;
> +		break;
> +	default:
> +		rc = -EINVAL;
> +	}
> +
> +out:
> +	pr_debug("rc=%d\n", rc);
> +	return rc;
> +}
> +
>
Herbert Xu Nov. 12, 2024, 1:34 a.m. UTC | #2 
On Mon, Nov 11, 2024 at 01:13:34PM +0100, Ingo Franzki wrote:
>
> There is no need to perform the (possibly time consuming) key conversation at every init.

The key is converted not just at init, but multiple times during
each update operation.  Remember the API is multi-threaded, so at
any time there could be multiple tfm's, each with a different key
being operated on in parallel.

If the hardware is such that you can only have one converted key,
then I'm afraid you're stuck with having to do the conversion before
each use.

However, since we are converting the key during each update, why
is it even stored in the tfm_ctx?

Cheers,
Herbert Xu Nov. 12, 2024, 1:38 a.m. UTC | #3 
On Thu, Nov 07, 2024 at 03:55:21PM +0100, Harald Freudenberger wrote:
>
> +static int s390_phmac_sha2_init(struct shash_desc *desc)
> +{
> +	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
> +	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
> +	unsigned int bs = crypto_shash_blocksize(desc->tfm);
> +	int rc;
> +
> +	rc = phmac_convert_key(desc->tfm);
> +	if (rc)
> +		goto out;
> +
> +	spin_lock_bh(&tfm_ctx->pk_lock);
> +	memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
> +	       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
> +	spin_unlock_bh(&tfm_ctx->pk_lock);

This appers to be completely broken.  Each tfm can be used by
an unlimited number of descriptors in parallel.  So you cannot
modify the tfm context.  I see that you have taken spinlocks
around it, but it is still broken:

CPU1			CPU2
lock(tfm)
tfm->pk = pk1
unlock(tfm)
			lock(tfm)
			tfm->pk = pk2
			unlock(tfm)
lock(tfm)
copy tfm->pk to desc
	pk2 is copied
unlock(tfm)

Now this could all be harmless because pk1 and pk2 may be guaranteed
to be the same, but if that's the case why go through all this in
the first place? You could've just done it in setkey.

Cheers,
Harald Freudenberger Nov. 12, 2024, 12:45 p.m. UTC | #4 
On 2024-11-12 02:38, Herbert Xu wrote:
> On Thu, Nov 07, 2024 at 03:55:21PM +0100, Harald Freudenberger wrote:
>> 
>> +static int s390_phmac_sha2_init(struct shash_desc *desc)
>> +{
>> +	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
>> +	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
>> +	unsigned int bs = crypto_shash_blocksize(desc->tfm);
>> +	int rc;
>> +
>> +	rc = phmac_convert_key(desc->tfm);
>> +	if (rc)
>> +		goto out;
>> +
>> +	spin_lock_bh(&tfm_ctx->pk_lock);
>> +	memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
>> +	       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
>> +	spin_unlock_bh(&tfm_ctx->pk_lock);
> 
> This appers to be completely broken.  Each tfm can be used by
> an unlimited number of descriptors in parallel.  So you cannot
> modify the tfm context.  I see that you have taken spinlocks
> around it, but it is still broken:
> 
> CPU1			CPU2
> lock(tfm)
> tfm->pk = pk1
> unlock(tfm)
> 			lock(tfm)
> 			tfm->pk = pk2
> 			unlock(tfm)
> lock(tfm)
> copy tfm->pk to desc
> 	pk2 is copied
> unlock(tfm)
> 
> Now this could all be harmless because pk1 and pk2 may be guaranteed
> to be the same, but if that's the case why go through all this in
> the first place? You could've just done it in setkey.
> 
> Cheers,

Well, we had a similar discussion once with paes (See
https://lore.kernel.org/linux-crypto/20191113105523.8007-1-freude@linux.ibm.com/)

The tfm holds the pkey which is a hardware wrapped version of the key 
value.
It is generated by a special invocation done via the PKEY kernel 
module(s) which
knows how to unpack the raw key material and re-wrap it so it can be 
used
with the CPACF instructions. The hardware wrapping key may change - in 
fact
it chances for example with a KVM guest relocated to another system and 
then
this unpack/rewrap cycle needs to be triggered again and thus the pkey 
may
change but the underlying "effective" or "real" key stays the same.
In that sense the tfm holding the pkey value is updated. To make the 
update
of the pkey atomic the spinlock is used as the tfm may be used by 
multiple
hash contexts.

Why not convert in the setkey() function? As of now this could be an 
option
as the invocation of convert_key() in the end within the PKEY pkey_pckmo
kernel module only calls PCKMO to generate the wrapped pkey. However, 
long
term we will have another path using a crypto card for this action and
then we are clearly in a sleeping context which must not be used from
setkey(). So make it correct now means to delay the conversion from 
setkey()
to later: the init of the hash context is the next chance to do this.

I see that calling the conversion each time a shash_init() is called
is total overkill and an avoidable action as the dm-integrity layer 
calls
this per sector. This may even get worse if we intent to go a hardware
path down to convert the key. So I am thinking of a better way which
avoids this overhead ... patch is under construction ...

Regards
Harald Freudenberger
Herbert Xu Nov. 14, 2024, 11:17 a.m. UTC | #5 
On Tue, Nov 12, 2024 at 01:45:53PM +0100, Harald Freudenberger wrote:
.
> Well, we had a similar discussion once with paes (See
> https://lore.kernel.org/linux-crypto/20191113105523.8007-1-freude@linux.ibm.com/)

Thanks for refreshing my memroy.
 
> Why not convert in the setkey() function? As of now this could be an option
> as the invocation of convert_key() in the end within the PKEY pkey_pckmo
> kernel module only calls PCKMO to generate the wrapped pkey. However, long
> term we will have another path using a crypto card for this action and
> then we are clearly in a sleeping context which must not be used from
> setkey(). So make it correct now means to delay the conversion from setkey()
> to later: the init of the hash context is the next chance to do this.

I'm not sure I get this justification for putting it in init instead
of setkey.  If you can't do it in setkey because it has to sleep,
then surely you can't do it from init/udpate either as they too
can be called from atomic contexts?

Cheers,
diff mbox series

Patch

diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 6b602d972e91..a8ba72e7162d 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -804,6 +804,7 @@  CONFIG_PKEY_EP11=m
 CONFIG_PKEY_PCKMO=m
 CONFIG_PKEY_UV=m
 CONFIG_CRYPTO_PAES_S390=m
+CONFIG_CRYPTO_PHMAC_S390=m
 CONFIG_CRYPTO_DEV_VIRTIO=m
 CONFIG_SYSTEM_BLACKLIST_KEYRING=y
 CONFIG_CORDIC=m
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index 7844b9f5851b..cd2e9c94b736 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -790,6 +790,7 @@  CONFIG_PKEY_EP11=m
 CONFIG_PKEY_PCKMO=m
 CONFIG_PKEY_UV=m
 CONFIG_CRYPTO_PAES_S390=m
+CONFIG_CRYPTO_PHMAC_S390=m
 CONFIG_CRYPTO_DEV_VIRTIO=m
 CONFIG_SYSTEM_BLACKLIST_KEYRING=y
 CONFIG_CORDIC=m
diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile
index a0cb96937c3d..47637140b95c 100644
--- a/arch/s390/crypto/Makefile
+++ b/arch/s390/crypto/Makefile
@@ -16,6 +16,7 @@  obj-$(CONFIG_S390_PRNG) += prng.o
 obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
 obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o
 obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
+obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o
 obj-y += arch_random.o
 
 crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o
diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c
new file mode 100644
index 000000000000..77a5244b2eb4
--- /dev/null
+++ b/arch/s390/crypto/phmac_s390.c
@@ -0,0 +1,473 @@ 
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright IBM Corp. 2024
+ *
+ * s390 specific HMAC support for protected keys.
+ */
+
+#define KMSG_COMPONENT	"phmac_s390"
+#define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
+
+#include <asm/cpacf.h>
+#include <asm/pkey.h>
+#include <crypto/sha2.h>
+#include <crypto/internal/hash.h>
+#include <linux/cpufeature.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+
+/*
+ * KMAC param block layout for sha2 function codes:
+ * The layout of the param block for the KMAC instruction depends on the
+ * blocksize of the used hashing sha2-algorithm function codes. The param block
+ * contains the hash chaining value (cv), the input message bit-length (imbl)
+ * and the hmac-secret (key). To prevent code duplication, the sizes of all
+ * these are calculated based on the blocksize.
+ *
+ * param-block:
+ * +-------+
+ * | cv    |
+ * +-------+
+ * | imbl  |
+ * +-------+
+ * | key   |
+ * +-------+
+ *
+ * sizes:
+ * part | sh2-alg | calculation | size | type
+ * -----+---------+-------------+------+--------
+ * cv	| 224/256 | blocksize/2 |   32 |  u64[8]
+ *	| 384/512 |		|   64 | u128[8]
+ * imbl | 224/256 | blocksize/8 |    8 |     u64
+ *	| 384/512 |		|   16 |    u128
+ * key	| 224/256 | blocksize	|   96 |  u8[96]
+ *	| 384/512 |		|  160 | u8[160]
+ */
+
+#define MAX_DIGEST_SIZE		SHA512_DIGEST_SIZE
+#define MAX_IMBL_SIZE		sizeof(u128)
+#define MAX_BLOCK_SIZE		SHA512_BLOCK_SIZE
+
+#define SHA2_CV_SIZE(bs)	((bs) >> 1)
+#define SHA2_IMBL_SIZE(bs)	((bs) >> 3)
+
+#define SHA2_IMBL_OFFSET(bs)	(SHA2_CV_SIZE(bs))
+#define SHA2_KEY_OFFSET(bs)	(SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))
+
+#define PHMAC_SHA256_KEY_SIZE	(SHA256_BLOCK_SIZE + 32)
+#define PHMAC_SHA512_KEY_SIZE	(SHA512_BLOCK_SIZE + 32)
+#define PHMAC_MAX_KEY_SIZE	PHMAC_SHA512_KEY_SIZE
+
+struct phmac_protkey {
+	u32 type;
+	u32 len;
+	u8 protkey[PHMAC_MAX_KEY_SIZE];
+};
+
+struct s390_phmac_ctx {
+	u8 *key;
+	unsigned int keylen;
+
+	struct phmac_protkey pk;
+	/* spinlock to atomic update pk */
+	spinlock_t pk_lock;
+};
+
+union s390_kmac_gr0 {
+	unsigned long reg;
+	struct {
+		unsigned long		: 48;
+		unsigned long ikp	:  1;
+		unsigned long iimp	:  1;
+		unsigned long ccup	:  1;
+		unsigned long		:  6;
+		unsigned long fc	:  7;
+	};
+};
+
+struct s390_kmac_sha2_ctx {
+	u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + PHMAC_MAX_KEY_SIZE];
+	union s390_kmac_gr0 gr0;
+	u8 buf[MAX_BLOCK_SIZE];
+	unsigned int buflen;
+};
+
+/*
+ * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
+ */
+static inline void kmac_sha2_set_imbl(u8 *param, unsigned int buflen,
+				      unsigned int blocksize)
+{
+	u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize);
+
+	switch (blocksize) {
+	case SHA256_BLOCK_SIZE:
+		*(u64 *)imbl = (u64)buflen * BITS_PER_BYTE;
+		break;
+	case SHA512_BLOCK_SIZE:
+		*(u128 *)imbl = (u128)buflen * BITS_PER_BYTE;
+		break;
+	default:
+		break;
+	}
+}
+
+static inline int phmac_keyblob2pkey(const u8 *key, unsigned int keylen,
+				     struct phmac_protkey *pk)
+{
+	int i, rc = -EIO;
+
+	/* try three times in case of busy card */
+	for (i = 0; rc && i < 3; i++) {
+		if (rc == -EBUSY && in_task()) {
+			if (msleep_interruptible(1000))
+				return -EINTR;
+		}
+		rc = pkey_key2protkey(key, keylen,
+				      pk->protkey, &pk->len, &pk->type);
+	}
+
+	return rc;
+}
+
+static inline int phmac_convert_key(struct crypto_shash *tfm)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(tfm);
+	struct phmac_protkey pk;
+	int rc;
+
+	pk.len = sizeof(pk.protkey);
+	rc = phmac_keyblob2pkey(tfm_ctx->key, tfm_ctx->keylen, &pk);
+	if (rc)
+		goto out;
+
+	rc = -EINVAL;
+	switch (crypto_shash_digestsize(tfm)) {
+	case SHA224_DIGEST_SIZE:
+	case SHA256_DIGEST_SIZE:
+		if (pk.type != PKEY_KEYTYPE_HMAC_512)
+			goto out;
+		break;
+	case SHA384_DIGEST_SIZE:
+	case SHA512_DIGEST_SIZE:
+		if (pk.type != PKEY_KEYTYPE_HMAC_1024)
+			goto out;
+		break;
+	default:
+		goto out;
+	}
+
+	spin_lock_bh(&tfm_ctx->pk_lock);
+	tfm_ctx->pk = pk;
+	spin_unlock_bh(&tfm_ctx->pk_lock);
+
+	rc = 0;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static inline int s390_phmac_sha2_setkey(struct crypto_shash *tfm,
+					 const u8 *key, unsigned int keylen)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(tfm);
+
+	if (tfm_ctx->keylen) {
+		kfree_sensitive(tfm_ctx->key);
+		tfm_ctx->key = NULL;
+		tfm_ctx->keylen = 0;
+	}
+
+	tfm_ctx->key = kmemdup(key, keylen, GFP_ATOMIC);
+	if (!tfm_ctx->key)
+		return -ENOMEM;
+	tfm_ctx->keylen = keylen;
+
+	return 0;
+}
+
+static int s390_phmac_sha2_init(struct shash_desc *desc)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
+	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
+	unsigned int bs = crypto_shash_blocksize(desc->tfm);
+	int rc;
+
+	rc = phmac_convert_key(desc->tfm);
+	if (rc)
+		goto out;
+
+	spin_lock_bh(&tfm_ctx->pk_lock);
+	memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
+	       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
+	spin_unlock_bh(&tfm_ctx->pk_lock);
+
+	ctx->buflen = 0;
+	ctx->gr0.reg = 0;
+
+	switch (crypto_shash_digestsize(desc->tfm)) {
+	case SHA224_DIGEST_SIZE:
+		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_224;
+		break;
+	case SHA256_DIGEST_SIZE:
+		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_256;
+		break;
+	case SHA384_DIGEST_SIZE:
+		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_384;
+		break;
+	case SHA512_DIGEST_SIZE:
+		ctx->gr0.fc = CPACF_KMAC_PHMAC_SHA_512;
+		break;
+	default:
+		rc = -EINVAL;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int s390_phmac_sha2_update(struct shash_desc *desc,
+				  const u8 *data, unsigned int len)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
+	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
+	unsigned int bs = crypto_shash_blocksize(desc->tfm);
+	unsigned int offset, n, k;
+
+	/* check current buffer */
+	offset = ctx->buflen % bs;
+	ctx->buflen += len;
+	if (offset + len < bs)
+		goto store;
+
+	/* process one stored block */
+	if (offset) {
+		n = bs - offset;
+		memcpy(ctx->buf + offset, data, n);
+		ctx->gr0.iimp = 1;
+		for (k = bs;;) {
+			k -= _cpacf_kmac(&ctx->gr0.reg, ctx->param,
+					 ctx->buf + bs - k, k);
+			if (!k)
+				break;
+			if (phmac_convert_key(desc->tfm))
+				return -EIO;
+			spin_lock_bh(&tfm_ctx->pk_lock);
+			memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
+			       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
+			spin_unlock_bh(&tfm_ctx->pk_lock);
+		}
+		data += n;
+		len -= n;
+		offset = 0;
+	}
+	/* process as many blocks as possible */
+	if (len >= bs) {
+		n = (len / bs) * bs;
+		ctx->gr0.iimp = 1;
+		for (k = n;;) {
+			k -= _cpacf_kmac(&ctx->gr0.reg, ctx->param,
+					 data + n - k, k);
+			if (!k)
+				break;
+			if (phmac_convert_key(desc->tfm))
+				return -EIO;
+			spin_lock_bh(&tfm_ctx->pk_lock);
+			memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
+			       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
+			spin_unlock_bh(&tfm_ctx->pk_lock);
+		}
+		data += n;
+		len -= n;
+	}
+store:
+	/* store incomplete block in buffer */
+	if (len)
+		memcpy(ctx->buf + offset, data, len);
+
+	return 0;
+}
+
+static int s390_phmac_sha2_final(struct shash_desc *desc, u8 *out)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
+	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
+	unsigned int bs = crypto_shash_blocksize(desc->tfm);
+	unsigned int n, k;
+
+	n = ctx->buflen % bs;
+	ctx->gr0.iimp = 0;
+	kmac_sha2_set_imbl(ctx->param, ctx->buflen, bs);
+	for (k = n;;) {
+		k -= _cpacf_kmac(&ctx->gr0.reg, ctx->param,
+				 ctx->buf + n - k, k);
+		if (!k)
+			break;
+		if (phmac_convert_key(desc->tfm))
+			return -EIO;
+		spin_lock_bh(&tfm_ctx->pk_lock);
+		memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
+		       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
+		spin_unlock_bh(&tfm_ctx->pk_lock);
+	}
+	memcpy(out, ctx->param, crypto_shash_digestsize(desc->tfm));
+
+	return 0;
+}
+
+static int s390_phmac_sha2_digest(struct shash_desc *desc,
+				  const u8 *data, unsigned int len, u8 *out)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm);
+	struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
+	unsigned int ds = crypto_shash_digestsize(desc->tfm);
+	unsigned int bs = crypto_shash_blocksize(desc->tfm);
+	unsigned int k;
+	int rc;
+
+	rc = s390_phmac_sha2_init(desc);
+	if (rc)
+		return rc;
+
+	ctx->gr0.iimp = 0;
+	kmac_sha2_set_imbl(ctx->param, len, bs);
+	for (k = len;;) {
+		k -= _cpacf_kmac(&ctx->gr0.reg, ctx->param,
+				 data + len - k, k);
+		if (!k)
+			break;
+		if (phmac_convert_key(desc->tfm))
+			return -EIO;
+		spin_lock_bh(&tfm_ctx->pk_lock);
+		memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
+		       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
+		spin_unlock_bh(&tfm_ctx->pk_lock);
+	}
+	memcpy(out, ctx->param, ds);
+
+	return 0;
+}
+
+static int s390_phmac_sha2_init_tfm(struct crypto_shash *tfm)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(tfm);
+
+	tfm_ctx->key = NULL;
+	tfm_ctx->keylen = 0;
+	spin_lock_init(&tfm_ctx->pk_lock);
+
+	return 0;
+}
+
+static void s390_phmac_sha2_exit_tfm(struct crypto_shash *tfm)
+{
+	struct s390_phmac_ctx *tfm_ctx = crypto_shash_ctx(tfm);
+
+	memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk));
+	kfree_sensitive(tfm_ctx->key);
+}
+
+static int s390_phmac_sha2_clone_tfm(struct crypto_shash *dst,
+				     struct crypto_shash *src)
+{
+	struct s390_phmac_ctx *src_ctx = crypto_shash_ctx(src);
+	int rc;
+
+	rc = s390_phmac_sha2_init_tfm(dst);
+	if (rc)
+		return rc;
+
+	if (src_ctx->key && src_ctx->keylen)
+		rc = s390_phmac_sha2_setkey(dst, src_ctx->key, src_ctx->keylen);
+
+	return rc;
+}
+
+#define S390_HMAC_SHA2_ALG(x) {						\
+	.fc = CPACF_KMAC_PHMAC_SHA_##x,					\
+	.alg = {							\
+		.init = s390_phmac_sha2_init,				\
+		.update = s390_phmac_sha2_update,			\
+		.final = s390_phmac_sha2_final,				\
+		.digest = s390_phmac_sha2_digest,			\
+		.setkey = s390_phmac_sha2_setkey,			\
+		.init_tfm = s390_phmac_sha2_init_tfm,			\
+		.exit_tfm = s390_phmac_sha2_exit_tfm,			\
+		.clone_tfm = s390_phmac_sha2_clone_tfm,			\
+		.descsize = sizeof(struct s390_kmac_sha2_ctx),		\
+		.halg = {						\
+			.digestsize = SHA##x##_DIGEST_SIZE,		\
+			.base = {					\
+				.cra_name = "phmac(sha" #x ")",		\
+				.cra_driver_name = "phmac_s390_sha" #x,	\
+				.cra_blocksize = SHA##x##_BLOCK_SIZE,	\
+				.cra_priority = 400,			\
+				.cra_ctxsize = sizeof(struct s390_phmac_ctx), \
+				.cra_module = THIS_MODULE,		\
+			},						\
+		},							\
+	},								\
+}
+
+static struct s390_hmac_alg {
+	bool registered;
+	unsigned int fc;
+	struct shash_alg alg;
+} s390_hmac_algs[] = {
+	S390_HMAC_SHA2_ALG(224),
+	S390_HMAC_SHA2_ALG(256),
+	S390_HMAC_SHA2_ALG(384),
+	S390_HMAC_SHA2_ALG(512),
+};
+
+static __always_inline void _s390_hmac_algs_unregister(void)
+{
+	struct s390_hmac_alg *hmac;
+	int i;
+
+	for (i = ARRAY_SIZE(s390_hmac_algs) - 1; i >= 0; i--) {
+		hmac = &s390_hmac_algs[i];
+		if (!hmac->registered)
+			continue;
+		crypto_unregister_shash(&hmac->alg);
+	}
+}
+
+static int __init phmac_s390_init(void)
+{
+	struct s390_hmac_alg *hmac;
+	int i, rc = -ENODEV;
+
+	for (i = 0; i < ARRAY_SIZE(s390_hmac_algs); i++) {
+		hmac = &s390_hmac_algs[i];
+		if (!cpacf_query_func(CPACF_KMAC, hmac->fc))
+			continue;
+
+		rc = crypto_register_shash(&hmac->alg);
+		if (rc) {
+			pr_err("unable to register %s\n",
+			       hmac->alg.halg.base.cra_name);
+			goto out;
+		}
+		hmac->registered = true;
+		pr_debug("registered %s\n", hmac->alg.halg.base.cra_name);
+	}
+	return rc;
+out:
+	_s390_hmac_algs_unregister();
+	return rc;
+}
+
+static void __exit phmac_s390_exit(void)
+{
+	_s390_hmac_algs_unregister();
+}
+
+module_init(phmac_s390_init);
+module_exit(phmac_s390_exit);
+
+MODULE_DESCRIPTION("S390 HMAC driver for protected keys");
+MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 0a9cdd31cbd9..519305e04f18 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -187,6 +187,18 @@  config CRYPTO_PAES_S390
 	  Select this option if you want to use the paes cipher
 	  for example to use protected key encrypted devices.
 
+config CRYPTO_PHMAC_S390
+	tristate "PHMAC cipher algorithms"
+	depends on S390
+	depends on PKEY
+	select CRYPTO_HASH
+	help
+	  This is the s390 hardware accelerated implementation of the
+	  protected key HMAC support for SHA224, SHA256, SHA384 and SHA512.
+
+	  Select this option if you want to use the phmac digests
+	  for example to use dm-integrity with secure/protected keys.
+
 config S390_PRNG
 	tristate "Pseudo random number generator device driver"
 	depends on S390