@@ -98,6 +98,7 @@ typedef
odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
odp_crypto_generic_session_t *session);
+typedef void (*crypto_init_func_t)(odp_crypto_generic_session_t *session);
/**
* Per crypto session data structure
@@ -117,6 +118,7 @@ struct odp_crypto_generic_session_t {
const EVP_CIPHER *evp_cipher;
crypto_func_t func;
+ crypto_init_func_t init;
} cipher;
struct {
@@ -128,6 +130,7 @@ struct odp_crypto_generic_session_t {
const EVP_CIPHER *evp_cipher;
};
crypto_func_t func;
+ crypto_init_func_t init;
} auth;
unsigned idx;
@@ -140,13 +143,9 @@ struct odp_crypto_global_s {
odp_crypto_generic_session_t *free;
odp_crypto_generic_session_t sessions[MAX_SESSIONS];
- /* These bitfields are cleared at alloc_session()
+ /* This bitfield is cleared at alloc_session()
* together with the rest of data */
- bitset_t hmac_valid[ODP_THREAD_COUNT_MAX]
- [(MAX_SESSIONS + ATOM_BITSET_SIZE - 1) / ATOM_BITSET_SIZE];
- bitset_t cipher_valid[ODP_THREAD_COUNT_MAX]
- [(MAX_SESSIONS + ATOM_BITSET_SIZE - 1) / ATOM_BITSET_SIZE];
- bitset_t mac_cipher_valid[ODP_THREAD_COUNT_MAX]
+ bitset_t ctx_valid[ODP_THREAD_COUNT_MAX]
[(MAX_SESSIONS + ATOM_BITSET_SIZE - 1) / ATOM_BITSET_SIZE];
odp_ticketlock_t openssl_lock[0];
@@ -158,30 +157,26 @@ typedef struct crypto_local_t {
HMAC_CTX *hmac_ctx[MAX_SESSIONS];
EVP_CIPHER_CTX *cipher_ctx[MAX_SESSIONS];
EVP_CIPHER_CTX *mac_cipher_ctx[MAX_SESSIONS];
- bitset_t *hmac_valid;
- bitset_t *cipher_valid;
- bitset_t *mac_cipher_valid;
+ bitset_t *ctx_valid;
} crypto_local_t;
static __thread crypto_local_t local;
typedef enum { KIND_HMAC, KIND_CIPHER, KIND_MAC_CIPHER } crypto_kind_t;
-static inline int crypto_should_init(odp_crypto_generic_session_t *session,
- crypto_kind_t kind)
+static inline void crypto_init(odp_crypto_generic_session_t *session)
{
- bitset_t *ptr = kind == KIND_HMAC ? local.hmac_valid :
- kind == KIND_CIPHER ? local.cipher_valid :
- local.mac_cipher_valid;
+ bitset_t *ptr = local.ctx_valid;
bitset_t *cv = ptr + (session->idx / ATOM_BITSET_SIZE);
bitset_t cur = atom_bitset_load(cv, __ATOMIC_ACQUIRE);
if (bitset_is_set(cur, session->idx % ATOM_BITSET_SIZE))
- return false;
+ return;
- atom_bitset_set(cv, session->idx % ATOM_BITSET_SIZE, __ATOMIC_RELEASE);
+ session->cipher.init(session);
+ session->auth.init(session);
- return true;
+ atom_bitset_set(cv, session->idx % ATOM_BITSET_SIZE, __ATOMIC_RELEASE);
}
static
@@ -203,17 +198,7 @@ odp_crypto_generic_session_t *alloc_session(void)
for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
bitset_t *cv;
- cv = global->hmac_valid[i] +
- (session->idx / ATOM_BITSET_SIZE);
- atom_bitset_clr(cv, session->idx % ATOM_BITSET_SIZE,
- __ATOMIC_ACQ_REL);
-
- cv = global->cipher_valid[i] +
- (session->idx / ATOM_BITSET_SIZE);
- atom_bitset_clr(cv, session->idx % ATOM_BITSET_SIZE,
- __ATOMIC_ACQ_REL);
-
- cv = global->mac_cipher_valid[i] +
+ cv = global->ctx_valid[i] +
(session->idx / ATOM_BITSET_SIZE);
atom_bitset_clr(cv, session->idx % ATOM_BITSET_SIZE,
__ATOMIC_ACQ_REL);
@@ -239,6 +224,12 @@ null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
return ODP_CRYPTO_ALG_ERR_NONE;
}
+static void
+null_crypto_init_routine(odp_crypto_generic_session_t *session ODP_UNUSED)
+{
+ return;
+}
+
/* Mimic new OpenSSL 1.1.y API */
#if OPENSSL_VERSION_NUMBER < 0x10100000L
static HMAC_CTX *HMAC_CTX_new(void)
@@ -256,6 +247,18 @@ static void HMAC_CTX_free(HMAC_CTX *ctx)
}
#endif
+static void
+auth_init(odp_crypto_generic_session_t *session)
+{
+ HMAC_CTX *ctx = local.hmac_ctx[session->idx];
+
+ HMAC_Init_ex(ctx,
+ session->auth.key,
+ session->auth.key_length,
+ session->auth.evp_md,
+ NULL);
+}
+
static
void packet_hmac(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -268,20 +271,8 @@ void packet_hmac(odp_packet_t pkt,
ODP_ASSERT(offset + len <= odp_packet_len(pkt));
- if (crypto_should_init(session, KIND_HMAC)) {
- HMAC_Init_ex(ctx,
- session->auth.key,
- session->auth.key_length,
- session->auth.evp_md,
- NULL);
- } else {
- /* Reinitialize HMAC calculation without resetting the key */
- HMAC_Init_ex(ctx,
- NULL,
- 0,
- NULL,
- NULL);
- }
+ /* Reinitialize HMAC calculation without resetting the key */
+ HMAC_Init_ex(ctx, NULL, 0, NULL, NULL);
/* Hash it */
while (len > 0) {
@@ -486,6 +477,16 @@ int internal_decrypt(EVP_CIPHER_CTX *ctx,
return ret;
}
+static void
+cipher_encrypt_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
+
+ EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
+ session->cipher.key_data, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -502,12 +503,6 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Encrypt it */
- if (crypto_should_init(session, KIND_CIPHER)) {
- EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
- session->cipher.key_data, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
ret = internal_encrypt(ctx, pkt, param);
@@ -516,6 +511,16 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt,
ODP_CRYPTO_ALG_ERR_NONE;
}
+static void
+cipher_decrypt_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
+
+ EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
+ session->cipher.key_data, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -532,12 +537,6 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Decrypt it */
- if (crypto_should_init(session, KIND_CIPHER)) {
- EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
- session->cipher.key_data, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
ret = internal_decrypt(ctx, pkt, param);
@@ -565,14 +564,29 @@ static int process_cipher_param(odp_crypto_generic_session_t *session,
session->p.cipher_key.length);
/* Set function */
- if (ODP_CRYPTO_OP_ENCODE == session->p.op)
+ if (ODP_CRYPTO_OP_ENCODE == session->p.op) {
session->cipher.func = cipher_encrypt;
- else
+ session->cipher.init = cipher_encrypt_init;
+ } else {
session->cipher.func = cipher_decrypt;
+ session->cipher.init = cipher_decrypt_init;
+ }
return 0;
}
+static void
+aes_gcm_encrypt_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
+
+ EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
+ session->cipher.key_data, NULL);
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
+ session->p.iv.length, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -593,14 +607,6 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Encrypt it */
- if (crypto_should_init(session, KIND_CIPHER)) {
- EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
- session->cipher.key_data, NULL);
- EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
- session->p.iv.length, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
/* Authenticate header data (if any) without encrypting them */
@@ -619,6 +625,18 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
ODP_CRYPTO_ALG_ERR_NONE;
}
+static void
+aes_gcm_decrypt_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
+
+ EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
+ session->cipher.key_data, NULL);
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
+ session->p.iv.length, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -639,14 +657,6 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Decrypt it */
- if (crypto_should_init(session, KIND_CIPHER)) {
- EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL,
- session->cipher.key_data, NULL);
- EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
- session->p.iv.length, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
odp_packet_copy_to_mem(pkt, param->hash_result_offset,
@@ -679,14 +689,29 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session,
session->cipher.evp_cipher = cipher;
/* Set function */
- if (ODP_CRYPTO_OP_ENCODE == session->p.op)
+ if (ODP_CRYPTO_OP_ENCODE == session->p.op) {
session->cipher.func = aes_gcm_encrypt;
- else
+ session->cipher.init = aes_gcm_encrypt_init;
+ } else {
session->cipher.func = aes_gcm_decrypt;
+ session->cipher.init = aes_gcm_decrypt_init;
+ }
return 0;
}
+static void
+aes_gmac_gen_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx];
+
+ EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL,
+ session->auth.key, NULL);
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
+ session->p.iv.length, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -704,14 +729,6 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Encrypt it */
- if (crypto_should_init(session, KIND_MAC_CIPHER)) {
- EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL,
- session->auth.key, NULL);
- EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
- session->p.iv.length, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
ret = internal_aad(ctx, pkt, param);
@@ -725,6 +742,18 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt,
ODP_CRYPTO_ALG_ERR_NONE;
}
+static void
+aes_gmac_check_init(odp_crypto_generic_session_t *session)
+{
+ EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx];
+
+ EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL,
+ session->auth.key, NULL);
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
+ session->p.iv.length, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+}
+
static
odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -742,14 +771,6 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt,
else
return ODP_CRYPTO_ALG_ERR_IV_INVALID;
- /* Decrypt it */
- if (crypto_should_init(session, KIND_MAC_CIPHER)) {
- EVP_DecryptInit_ex(ctx, session->auth.evp_cipher, NULL,
- session->auth.key, NULL);
- EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN,
- session->p.iv.length, NULL);
- EVP_CIPHER_CTX_set_padding(ctx, 0);
- }
EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
odp_packet_copy_to_mem(pkt, param->hash_result_offset,
@@ -779,10 +800,13 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session,
session->auth.evp_cipher = cipher;
/* Set function */
- if (ODP_CRYPTO_OP_ENCODE == session->p.op)
+ if (ODP_CRYPTO_OP_ENCODE == session->p.op) {
session->auth.func = aes_gmac_gen;
- else
+ session->auth.init = aes_gmac_gen_init;
+ } else {
session->auth.func = aes_gmac_check;
+ session->auth.init = aes_gmac_check_init;
+ }
return 0;
}
@@ -796,6 +820,7 @@ static int process_auth_param(odp_crypto_generic_session_t *session,
session->auth.func = auth_gen;
else
session->auth.func = auth_check;
+ session->auth.init = auth_init;
session->auth.evp_md = evp_md;
@@ -978,6 +1003,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param,
switch (param->cipher_alg) {
case ODP_CIPHER_ALG_NULL:
session->cipher.func = null_crypto_routine;
+ session->cipher.init = null_crypto_init_routine;
rc = 0;
break;
case ODP_CIPHER_ALG_3DES_CBC:
@@ -1053,6 +1079,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param,
switch (param->auth_alg) {
case ODP_AUTH_ALG_NULL:
session->auth.func = null_crypto_routine;
+ session->auth.init = null_crypto_init_routine;
rc = 0;
break;
#if ODP_DEPRECATED_API
@@ -1092,6 +1119,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param,
* cipher at the same time */
if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || aes_gcm) {
session->auth.func = null_crypto_routine;
+ session->auth.init = null_crypto_init_routine;
rc = 0;
} else {
rc = -1;
@@ -1293,12 +1321,8 @@ int _odp_crypto_init_local(void)
/* Clear local valid bits */
id = odp_thread_id();
- local.hmac_valid = global->hmac_valid[id];
- local.cipher_valid = global->cipher_valid[id];
- local.mac_cipher_valid = global->mac_cipher_valid[id];
- memset(local.hmac_valid, 0, sizeof(global->hmac_valid[0]));
- memset(local.cipher_valid, 0, sizeof(global->cipher_valid[0]));
- memset(local.mac_cipher_valid, 0, sizeof(global->mac_cipher_valid[0]));
+ local.ctx_valid = global->ctx_valid[id];
+ memset(local.ctx_valid, 0, sizeof(global->ctx_valid[0]));
return 0;
}
@@ -1488,6 +1512,8 @@ int odp_crypto_int(odp_packet_t pkt_in,
pkt_in = ODP_PACKET_INVALID;
}
+ crypto_init(session);
+
/* Invoke the functions */
if (session->do_cipher_first) {
rc_cipher = session->cipher.func(out_pkt, param, session);